Commit d60bc771 authored by Michal Berger's avatar Michal Berger Committed by Tomasz Zawadzki
Browse files

test/setup: Add tests for hugepages allocation 



These tests are meant to "predict" the right number of hugepages
given setup.sh instance will set, depending on its environment
options (HUGENODE, HUGEMEM, etc.).

Change-Id: Iaf441e69d25dd59941891e1d5c3c2b0dbd77644d
Signed-off-by: default avatarMichal Berger <michalx.berger@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5309


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarKarol Latecki <karol.latecki@intel.com>
parent ec94874c

test/setup/common.sh

+24 −0
Original line number Diff line number Diff line
@@ -7,3 +7,27 @@ setup() { 
		"$rootdir/scripts/setup.sh" "$@" &> /dev/null

	fi

}



get_meminfo() {

	xtrace_disable



	local get=$1

	local node=$2

	local var val

	local mem_f mem



	mem_f=/proc/meminfo

	if [[ -e /sys/devices/system/node/node$node/meminfo ]]; then

		mem_f=/sys/devices/system/node/node$node/meminfo

	fi

	mapfile -t mem < "$mem_f"

	mem=("${mem[@]#Node +([0-9]) }")



	while IFS=": " read -r var val _; do

		[[ $var == "$get" ]] || continue

		echo "$val" && return 0

	done < <(printf '%s\n' "${mem[@]}")

	return 1



	xtrace_restore

}

test/setup/hugepages.sh

0 → 100755
+187 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash

testdir=$(readlink -f "$(dirname "$0")")

rootdir=$(readlink -f "$testdir/../../")

source "$testdir/common.sh"



shopt -s extglob nullglob



declare -a nodes_sys=()



declare -i default_huges=0

declare -i no_nodes=0

declare -i nr_hugepages=0



default_huges=$(get_meminfo Hugepagesize)

default_huge_nr=/sys/kernel/mm/hugepages/hugepages-${default_huges}kB/nr_hugepages

global_huge_nr=/proc/sys/vm/nr_hugepages



get_nodes() {

	local node



	for node in /sys/devices/system/node/node+([0-9]); do

		nodes_sys[${node##*node}]=$(< "$node/hugepages/hugepages-${default_huges}kB/nr_hugepages")

	done

	no_nodes=${#nodes_sys[@]}

	((no_nodes > 0))

}



clear_hp() {

	local node hp



	for node in "${!nodes_sys[@]}"; do

		for hp in "/sys/devices/system/node/node$node/hugepages/hugepages-"*; do

			echo 0 > "$hp/nr_hugepages"

		done

	done



	export CLEAR_HUGE=yes

}



get_test_nr_hugepages() {

	local size=$1 # kB

	if (($# > 1)); then

		shift

		local node_ids=("$@")

	fi



	((size >= default_huges))



	nr_hugepages=$(((size + default_huges - 1) / default_huges))

	get_test_nr_hugepages_per_node "${node_ids[@]}"

}



get_test_nr_hugepages_per_node() {

	local user_nodes=("$@")



	local _nr_hugepages=$nr_hugepages

	local _no_nodes=$no_nodes



	local -g nodes_test=()



	if ((${#user_nodes[@]} > 0)); then

		for _no_nodes in "${user_nodes[@]}"; do

			nodes_test[_no_nodes]=$nr_hugepages

		done

		return 0

	elif ((${#nodes_hp[@]} > 0)); then

		for _no_nodes in "${!nodes_hp[@]}"; do

			nodes_test[_no_nodes]=${nodes_hp[_no_nodes]}

		done

		return 0

	fi



	while ((_no_nodes > 0)); do

		nodes_test[_no_nodes - 1]=$((_nr_hugepages / _no_nodes))

		: $((_nr_hugepages -= nodes_test[_no_nodes - 1]))

		: $((--_no_nodes))

	done

}



verify_nr_hugepages() {

	local node

	local sorted_t

	local sorted_s



	echo "nr_hugepages=$nr_hugepages"

	(($(< "$default_huge_nr") == nr_hugepages))

	(($(< "$global_huge_nr") == nr_hugepages))

	(($(get_meminfo HugePages_Total) == nr_hugepages))



	get_nodes



	# There's no obvious way of determining which NUMA node is going to end

	# up with an odd number of hugepages in case such number was actually

	# allocated by the kernel. Considering that, let's simply check if our

	# expaction is met by sorting and comparing it with nr of hugepages that

	# was actually allocated on each node.



	for node in "${!nodes_test[@]}"; do

		sorted_t[nodes_test[node]]=1 sorted_s[nodes_sys[node]]=1

		echo "node$node=${nodes_sys[node]}"

	done

	[[ ${!sorted_s[*]} == "${!sorted_t[*]}" ]]

}



# Test cases

default_setup() {

	# Default HUGEMEM (8G) alloc on node0

	get_test_nr_hugepages $((HUGEMEM * 1024)) 0

	setup

	verify_nr_hugepages

}



per_node_2G_alloc() {

	# 2G alloc per node, total N*2G pages

	local IFS=","



	get_test_nr_hugepages $((2048 * 1024)) "${!nodes_sys[@]}"

	NRHUGE=$nr_hugepages HUGENODE="${!nodes_sys[*]}" setup

	nr_hugepages=$((nr_hugepages * ${#nodes_sys[@]})) verify_nr_hugepages

}



even_2G_alloc() {

	# 2G alloc spread across N nodes

	get_test_nr_hugepages $((2048 * 1024))

	NRHUGE=$nr_hugepages HUGE_EVEN_ALLOC=yes setup

	verify_nr_hugepages

}



odd_alloc() {

	# Odd 2049MB alloc across N nodes

	get_test_nr_hugepages $((2049 * 1024))

	HUGEMEM=2049 HUGE_EVEN_ALLOC=yes setup

	verify_nr_hugepages

}



custom_alloc() {

	# Custom alloc: node0 == 512 pages [node1 == 1024 pages]



	local IFS=","



	local node

	local nodes_hp=()



	local nr_hugepages=0



	nodes_hp[0]=512

	if ((${#nodes_sys[@]} > 1)); then

		nodes_hp[1]=1024

	fi



	for node in "${!nodes_hp[@]}"; do

		HUGENODE+=("nodes_hp[$node]=${nodes_hp[node]}")

		((nr_hugepages += nodes_hp[node]))

	done



	get_test_nr_hugepages_per_node

	HUGENODE="${HUGENODE[*]}" setup

	verify_nr_hugepages

}



hp_status() {

	# Parse status from last verification



	local node

	local size free total



	((${#nodes_sys[@]} > 0))



	while read -r node size free _ total; do

		size=${size/kB/} node=${node#node}

		((size == default_huges)) || continue

		((free == nodes_test[node]))

		((total == nodes_test[node]))

	done < <(setup output status |& grep "node[0-9]")

}



get_nodes

clear_hp



run_test "default_setup" default_setup

run_test "per_node_2G_alloc" per_node_2G_alloc

run_test "even_2G_alloc" even_2G_alloc

run_test "odd_alloc" odd_alloc

run_test "custom_alloc" custom_alloc

run_test "hp_status" hp_status



clear_hp

test/setup/test-setup.sh

+1 −0
Original line number Diff line number Diff line
@@ -6,3 +6,4 @@ source "$testdir/common.sh" 
[[ $(uname -s) == Linux ]] || exit 0



run_test "acl" "$testdir/acl.sh"

run_test "hugepages" "$testdir/hugepages.sh"