Commit 972201d7 authored by Mateusz Kozlowski's avatar Mateusz Kozlowski Committed by Ben Walker
Browse files

test/ftl: Add upgrade shutdown test 



Change-Id: Icf3e9add29744cb7ab404593cee861d8c9790fc5
Signed-off-by: default avatarMariusz Barczak <Mariusz.Barczak@solidigmtechnology.com>
Signed-off-by: default avatarMateusz Kozlowski <mateusz.kozlowski@solidigm.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/19134


Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 28fef38c

test/ftl/common.sh

+153 −13
Original line number Diff line number Diff line
@@ -5,10 +5,29 @@ 


# Common utility functions to be sourced by the libftl test scripts



testdir=$(readlink -f "$(dirname "$0")")

rootdir=$(readlink -f "$testdir/../..")

rpc_py=$rootdir/scripts/rpc.py



export ftl_tgt_core_mask="[0]"



export spdk_tgt_bin="$SPDK_BIN_DIR/spdk_tgt"

export spdk_tgt_cpumask="[0]"

export spdk_tgt_cnfg="${testdir}/config/tgt.json"

export spdk_tgt_pid=""



export spdk_ini_bin="$SPDK_BIN_DIR/spdk_tgt"

export spdk_ini_cpumask="[1]"

export spdk_ini_rpc="/var/tmp/spdk.tgt.sock"

export spdk_ini_cnfg="${testdir}/config/ini.json"

export spdk_ini_pid=""



export spdk_dd_bin="$SPDK_BIN_DIR/spdk_dd"



function clear_lvols() {

	stores=$("$rootdir/scripts/rpc.py" bdev_lvol_get_lvstores | jq -r ".[] | .uuid")

	stores=$($rpc_py bdev_lvol_get_lvstores | jq -r ".[] | .uuid")

	for lvs in $stores; do

		"$rootdir/scripts/rpc.py" bdev_lvol_delete_lvstore -u $lvs

		$rpc_py bdev_lvol_delete_lvstore -u $lvs

	done

}
@@ -16,20 +35,19 @@ function create_nv_cache_bdev() { 
	local name=$1

	local cache_bdf=$2

	local base_bdev=$3

	local cache_size=$4



	# use 5% space of base bdev

	local size=$(($(get_bdev_size $base_bdev) * 5 / 100))

	local base_size=$(($(get_bdev_size "$base_bdev") * 5 / 100))



	# Create NVMe bdev on specified device and split it so that it has the desired size

	local nvc_bdev

	nvc_bdev=$($rootdir/scripts/rpc.py bdev_nvme_attach_controller -b $name -t PCIe -a $cache_bdf)

	nvc_bdev=$($rpc_py bdev_nvme_attach_controller -b "$name" -t PCIe -a "$cache_bdf")



	local nvc_size

	nvc_size=$(get_bdev_size $nvc_bdev)

	if [[ $size -gt $nvc_size ]]; then

		size=nvc_size

	if [[ -z "$cache_size" ]]; then

		cache_size=$(($(get_bdev_size "$base_bdev") * 5 / 100))

	fi

	$rootdir/scripts/rpc.py bdev_split_create $nvc_bdev -s $size 1

	$rpc_py bdev_split_create "$nvc_bdev" -s "$cache_size" 1

}



function create_base_bdev() {
@@ -39,17 +57,139 @@ function create_base_bdev() { 


	# Create NVMe bdev on specified device and split it so that it has the desired size

	local base_bdev

	base_bdev=$($rootdir/scripts/rpc.py bdev_nvme_attach_controller -b $name -t PCIe -a $base_bdf)

	base_bdev=$($rpc_py bdev_nvme_attach_controller -b $name -t PCIe -a $base_bdf)



	local base_size

	base_size=$(get_bdev_size $base_bdev)

	if [[ $size -le $base_size ]]; then

		$rootdir/scripts/rpc.py bdev_split_create $base_bdev -s $size 1

		$rpc_py bdev_split_create $base_bdev -s $size 1

	else

		clear_lvols

		lvs=$($rootdir/scripts/rpc.py bdev_lvol_create_lvstore $base_bdev lvs)

		$rootdir/scripts/rpc.py bdev_lvol_create ${base_bdev}p0 $size -t -u $lvs

		lvs=$($rpc_py bdev_lvol_create_lvstore $base_bdev lvs)

		$rpc_py bdev_lvol_create ${base_bdev}p0 $size -t -u $lvs

	fi

}



# required input variables:

# FTL_BDEV - FTL bdev name

# FTL_BASE - FTL base device PCIe path

# FTL_BASE_SIZE - FTL base device size

# FTL_CACHE - FTL NV cache device PCIe path

# FTL_CACHE_SIZE - FTL NV cache device size

# FTL_L2P_DRAM_LIMIT - FTL L2P DRAM limit

function tcp_target_setup() {

	local base_bdev=""

	local cache_bdev=""



	if [ -f "$spdk_tgt_cnfg" ]; then

		$spdk_tgt_bin "--cpumask=$spdk_tgt_cpumask" --config="$spdk_tgt_cnfg" &

	else

		$spdk_tgt_bin "--cpumask=$spdk_tgt_cpumask" &

	fi

	spdk_tgt_pid=$!

	export spdk_tgt_pid

	waitforlisten "$spdk_tgt_pid"



	if [[ -f "$spdk_tgt_cnfg" ]]; then

		# Configuration loaded from the JSON file

		return 0

	fi



	# Check if input parameters are available

	local params=(FTL_BDEV FTL_BASE FTL_BASE_SIZE FTL_CACHE FTL_CACHE_SIZE FTL_L2P_DRAM_LIMIT)

	for param in "${params[@]}"; do

		if [[ -z "${!param}" ]]; then

			echo "Missing $param"

			exit 1

		fi

	done



	base_bdev=$(create_base_bdev base "$FTL_BASE" "$FTL_BASE_SIZE")

	if [[ -z "$base_bdev" ]]; then

		echo "Cannot create base device"

		exit 1

	fi



	cache_bdev=$(create_nv_cache_bdev cache "$FTL_CACHE" "$base_bdev" "$FTL_CACHE_SIZE")

	if [[ -z "$cache_bdev" ]]; then

		echo "Cannot create nv cache device"

		exit 1

	fi



	$rpc_py -t 60 bdev_ftl_create -b "$FTL_BDEV" -d "$base_bdev" -c "$cache_bdev" --l2p_dram_limit "$FTL_L2P_DRAM_LIMIT"



	$rpc_py nvmf_create_transport --trtype TCP

	$rpc_py nvmf_create_subsystem nqn.2018-09.io.spdk:cnode0 -a -m 1

	$rpc_py nvmf_subsystem_add_ns nqn.2018-09.io.spdk:cnode0 "$FTL_BDEV"

	$rpc_py nvmf_subsystem_add_listener nqn.2018-09.io.spdk:cnode0 -t TCP -f ipv4 -s 4420 -a 127.0.0.1



	$rpc_py save_config > "$spdk_tgt_cnfg"

}



function tcp_target_shutdown() {

	if [[ -n "$spdk_tgt_pid" ]]; then

		killprocess "$spdk_tgt_pid"

		unset spdk_tgt_pid

	fi

}



function tcp_target_cleanup() {

	tcp_target_shutdown

	rm -f "$spdk_tgt_cnfg"

}



# required input variables:

# FTL_BDEV - FTL bdev name

function tcp_initiator_setup() {

	local rpc="$rpc_py -s ${spdk_ini_rpc}"



	if [[ -f "$spdk_ini_cnfg" ]]; then

		return 0

	fi



	if [[ -z "$FTL_BDEV" ]]; then

		echo "Missing FTL_BDEV"

		exit 1

	fi



	$spdk_ini_bin --cpumask="$spdk_ini_cpumask" --rpc-socket="$spdk_ini_rpc" &

	spdk_ini_pid=$!

	export spdk_ini_pid

	waitforlisten $spdk_ini_pid $spdk_ini_rpc



	$rpc bdev_nvme_attach_controller -b "$FTL_BDEV" \

		-t tcp -a 127.0.0.1 -s 4420 -f ipv4 -n nqn.2018-09.io.spdk:cnode0



	(

		echo '{"subsystems": ['

		$rpc save_subsystem_config -n bdev

		echo ']}'

	) > "$spdk_ini_cnfg"



	killprocess $spdk_ini_pid

	unset spdk_ini_pid

}



function tcp_initiator_shutdown() {

	if [[ -n "$spdk_ini_pid" ]]; then

		killprocess "$spdk_ini_pid"

		unset spdk_ini_pid

	fi

}



function tcp_initiator_cleanup() {

	tcp_initiator_shutdown

	rm -f "$spdk_ini_cnfg"

}



function tcp_cleanup() {

	tcp_target_cleanup

	tcp_initiator_cleanup

}



function tcp_dd() {

	tcp_initiator_setup

	$spdk_dd_bin --cpumask="$spdk_ini_cpumask" --rpc-socket="$spdk_ini_rpc" --json="$spdk_ini_cnfg" "$@"

}



# Remove not needed files from shared memory

test/ftl/ftl.sh

+1 −0
Original line number Diff line number Diff line
@@ -76,6 +76,7 @@ if [[ -z $SPDK_TEST_FTL_NIGHTLY ]]; then 
	run_test "ftl_trim" $testdir/trim.sh $device $nv_cache

	run_test "ftl_restore" $testdir/restore.sh -c $nv_cache $device

	run_test "ftl_dirty_shutdown" $testdir/dirty_shutdown.sh -c $nv_cache $device

	run_test "ftl_upgrade_shutdown" $testdir/upgrade_shutdown.sh $device $nv_cache

fi



if [ $SPDK_TEST_FTL_EXTENDED -eq 1 ]; then

test/ftl/upgrade_shutdown.sh

0 → 100755
+103 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash

#  SPDX-License-Identifier: BSD-3-Clause

#  Copyright 2023 Solidigm All Rights Reserved

#

testdir=$(readlink -f "$(dirname "$0")")

rootdir=$(readlink -f "$testdir/../..")

source "$rootdir/test/common/autotest_common.sh"

source "$testdir/common.sh"



function cleanup() {

	trap - SIGINT SIGTERM EXIT

	rm -f "${testdir}"/file

	rm -f "${testdir}"/file.md5

	tcp_cleanup

	remove_shm

}

trap "cleanup; exit 1" SIGINT SIGTERM EXIT



export FTL_BDEV=ftl

export FTL_BASE=$1

export FTL_BASE_SIZE=$((20 * 1024))

export FTL_CACHE=$2

export FTL_CACHE_SIZE=$((5 * 1024))

export FTL_L2P_DRAM_LIMIT=$((FTL_BASE_SIZE * 10 / 100 / 1024))



tcp_target_setup



size=$((2 ** 30)) # 1GiB

seek=0

skip=0

bs=$((2 ** 20)) # 1MiB

count=$((size / bs))

iterations=2

qd=2

sums=()



# Fill FTL

for ((i = 0; i < iterations; i++)); do

	echo "Fill FTL, iteration $((i + 1))"

	tcp_dd --if=/dev/urandom --ob=ftln1 --bs="$bs" --count="$count" --qd="$qd" --seek="$seek"

	seek=$((seek + count))



	echo "Calculate MD5 checksum, iteration $((i + 1))"

	tcp_dd --ib=ftln1 --of="${testdir}/file" --bs="$bs" --count="$count" --qd="$qd" --skip="$skip"

	skip=$((skip + count))



	md5sum "${testdir}/file" > "${testdir}/file.md5"

	sums[i]=$(cut -f1 -d' ' < "${testdir}/file.md5")

done



# Get properties of chunks

$rpc_py bdev_ftl_set_property -b ftl -p verbose_mode -v true

$rpc_py bdev_ftl_get_properties -b ftl



# Enable upgrade shutdown

$rpc_py bdev_ftl_set_property -b ftl -p prep_upgrade_on_shutdown -v true



# Validate there are utilized chunks

used=$($rpc_py bdev_ftl_get_properties -b ftl \

	| jq '[.properties[] | select(.name == "cache_device") | .chunks[] | select(.utilization != 0.0)] | length')

if [[ "$used" -eq 0 ]]; then

	echo "Shutdown upgrade ERROR, excepted utilized chunks"

	exit 1

fi



# Get properties of chunks

$rpc_py bdev_ftl_set_property -b ftl -p verbose_mode -v true

$rpc_py bdev_ftl_get_properties -b ftl



# Restart for upgrading

tcp_target_shutdown

tcp_target_setup



# Get properties of chunks

$rpc_py bdev_ftl_set_property -b ftl -p verbose_mode -v true

$rpc_py bdev_ftl_get_properties -b ftl



# Validate if all chunks utilization is 0.0

used=$($rpc_py bdev_ftl_get_properties -b ftl \

	| jq '[.properties[] | select(.name == "cache_device") | .chunks[] | select(.utilization != 0.0)] | length')

if [[ "$used" -ne 0 ]]; then

	echo "Shutdown upgrade ERROR, excepted only empty chunks"

	exit 1

fi



# Validate MD5 checksum

skip=0

for ((i = 0; i < iterations; i++)); do

	echo "Validate MD5 checksum, iteration $((i + 1))"

	tcp_dd --ib=ftln1 --of="${testdir}/file" --bs="$bs" --count="$count" --qd="$qd" --skip="$skip"

	skip=$((skip + count))



	md5sum "${testdir}/file" > "${testdir}/file.md5"

	sum=$(cut -f1 -d' ' < "${testdir}/file.md5")



	if [[ "${sums[$i]}" != "$sum" ]]; then

		echo "Wow, wow, wow!!! What is going on? MD5 Checksum ERROR!!!"

		exit 1

	fi

done



trap - SIGINT SIGTERM EXIT

cleanup