Commit 5c9a5039 authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Ben Walker
Browse files

test/ftl: use non-volatile cache in functional tests 



This patch adds non-volatile cache to some of the test configurations if
required device exists in the system (regular NVMe disk with separate
metadata support).

Change-Id: I0ea43990b360712361f34aeeb1982755f48b4dc5
Signed-off-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/459624


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 28439890

test/ftl/common.sh

0 → 100644
+35 −0
Original line number Diff line number Diff line 
# Common utility functions to be sourced by the libftl test scripts



function get_chunk_size() {

	echo $($rootdir/examples/nvme/identify/identify -r "trtype:PCIe traddr:$1" | \

		grep 'Logical blks per chunk' | sed 's/[^0-9]//g')

}



function has_separate_md() {

	local md_type=$($rootdir/examples/nvme/identify/identify -r "trtype:PCIe traddr:$1" | \

		grep 'Metadata Transferred' | cut -d: -f2)

	if [[ "$md_type" =~ Separate ]]; then

		return 0

	else

		return 1

	fi

}



function create_nv_cache_bdev() {

	local name=$1

	local ocssd_bdf=$2

	local cache_bdf=$3

	local num_punits=$4



	local bytes_to_mb=$[1024 * 1024]

	local chunk_size=$(get_chunk_size $ocssd_bdf)



	# We need at least 2 bands worth of data + 1 block

	local size=$[2 * 4096 * $chunk_size * $num_punits + 1]

	# Round the size up to the nearest megabyte

	local size=$[($size + $bytes_to_mb) / $bytes_to_mb]



	# Create NVMe bdev on specified device and split it so that it has the desired size

	local nvc_bdev=$($rootdir/scripts/rpc.py construct_nvme_bdev -b $name -t PCIe -a $cache_bdf)

	$rootdir/scripts/rpc.py construct_split_vbdev $nvc_bdev -s $size 1

}

test/ftl/dirty_shutdown.sh

+32 −43
Original line number Diff line number Diff line
@@ -3,19 +3,25 @@ 
testdir=$(readlink -f $(dirname $0))

rootdir=$(readlink -f $testdir/../..)

source $rootdir/test/common/autotest_common.sh

source $testdir/common.sh



rpc_py=$rootdir/scripts/rpc.py

pu_start=0

pu_end=3

additional_blocks=16



while getopts ':u:c:' opt; do

	case $opt in

		u) uuid=$OPTARG ;;

		c) nv_cache=$OPTARG ;;

		?) echo "Usage: $0 [-u UUID] [-c NV_CACHE_PCI_BDF] OCSSD_PCI_BDF" && exit 1 ;;

	esac

done

shift $((OPTIND -1))



device=$1

uuid=$2



restore_kill() {

	rm -f $testdir/config/ftl.json

	rm -f $testdir/empty

	rm -f $testdir/testblock

	rm -f $testdir/testfile.md5

	rm -f $testdir/testfile2.md5
@@ -26,22 +32,26 @@ restore_kill() { 


trap "restore_kill; exit 1" SIGINT SIGTERM EXIT



# Extract chunk size

chunk_size=$($rootdir/examples/nvme/identify/identify -r "trtype:PCIe traddr:$device" |

grep 'Logical blks per chunk' | sed 's/[^0-9]//g')

chunk_size=$(get_chunk_size $device)

pu_count=$(($pu_end - $pu_start + 1))



band_size=$(($chunk_size*($pu_end-$pu_start+1)))

# Write one band worth of data + one extra chunk

data_size=$(($chunk_size * ($pu_count + 1)))



$rootdir/app/spdk_tgt/spdk_tgt & svcpid=$!

# Wait until spdk_tgt starts

waitforlisten $svcpid



if [ -n "$uuid" ]; then

	$rpc_py construct_ftl_bdev -b nvme0 -a $device -l $pu_start-$pu_end -u $uuid -o

else

	$rpc_py construct_ftl_bdev -b nvme0 -a $device -l $pu_start-$pu_end -o

if [ -n "$nv_cache" ]; then

	nvc_bdev=$(create_nv_cache_bdev nvc0 $device $nv_cache $pu_count)

fi



ftl_construct_args="construct_ftl_bdev -b nvme0 -a $device -l $pu_start-$pu_end -o"



[ -n "$nvc_bdev" ] && ftl_construct_args+=" -c $nvc_bdev"

[ -n "$uuid" ]     && ftl_construct_args+=" -u $uuid"



$rpc_py $ftl_construct_args



# Load the nbd driver

modprobe nbd

$rpc_py start_nbd_disk nvme0 /dev/nbd0
@@ -49,53 +59,32 @@ waitfornbd nbd0 


$rpc_py save_config > $testdir/config/ftl.json



# Send band worth of data in 2 steps (some data should be written to 2nd band due to metadata overhead)

dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$(($band_size - $chunk_size)) oflag=dsync

offset=$(($band_size - $chunk_size))



dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$chunk_size seek=$offset oflag=dsync

offset=$(($offset + $chunk_size))



# Save md5 data of first batch (which should be fully on a closed band and recoverable)

dd if=/dev/nbd0 bs=4K count=$(($band_size - $chunk_size)) | md5sum > $testdir/testfile.md5



# Make sure the third batch of written data is fully on the second band

dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$additional_blocks seek=$offset oflag=dsync

offset=$(($offset + $additional_blocks))



dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$data_size oflag=dsync

# Calculate checksum of the data written

dd if=/dev/nbd0 bs=4K count=$data_size | md5sum > $testdir/testfile.md5

$rpc_py stop_nbd_disk /dev/nbd0



# Force kill bdev service (dirty shutdown) and start it again

kill -9 $svcpid

rm -f /dev/shm/spdk_tgt_trace.pid$svcpid

# TODO Adapt this after waitforlisten is expanded

sleep 5



$rootdir/app/spdk_tgt/spdk_tgt & svcpid=$!

# Wait until spdk_tgt starts

$rootdir/app/spdk_tgt/spdk_tgt -L ftl_init & svcpid=$!

waitforlisten $svcpid



# Ftl should recover, though with a loss of data (-o config option)

$rpc_py load_config < $testdir/config/ftl.json

waitfornbd nbd0



# Write extra data after restore

dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$chunk_size seek=$offset oflag=dsync

dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$chunk_size seek=$data_size oflag=dsync

# Save md5 data

dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$offset | md5sum > $testdir/testfile2.md5

dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$data_size | md5sum > $testdir/testfile2.md5



# Make sure all data will be read from disk

echo 3 > /proc/sys/vm/drop_caches



# Without persistent cache, first batch of data should be recoverable

dd if=/dev/nbd0 bs=4K count=$(($band_size - $chunk_size)) | md5sum -c $testdir/testfile.md5



dd if=/dev/nbd0 of=$testdir/testblock bs=4k count=$additional_blocks skip=$band_size

# Last 4k blocks from before restore should be on second band, and return as 0s

dd if=/dev/zero of=$testdir/empty bs=4k count=$additional_blocks

cmp $testdir/empty $testdir/testblock

# Verify data written after restore is still there

dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$(($band_size + $additional_blocks)) | md5sum -c $testdir/testfile2.md5

# Verify that the checksum matches and the data is consistent

dd if=/dev/nbd0 bs=4K count=$data_size | md5sum -c $testdir/testfile.md5

dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$data_size | md5sum -c $testdir/testfile2.md5



report_test_completion ftl_dirty_shutdown

test/ftl/ftl.sh

+25 −3
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
testdir=$(readlink -f $(dirname $0))

rootdir=$(readlink -f $testdir/../..)

source $rootdir/test/common/autotest_common.sh

source $testdir/common.sh



rpc_py=$rootdir/scripts/rpc.py
@@ -28,6 +29,22 @@ trap "at_ftl_exit" SIGINT SIGTERM EXIT 
# OCSSD is blacklisted so bind it to vfio/uio driver before testing

PCI_WHITELIST="$device" PCI_BLACKLIST="" DRIVER_OVERRIDE="" ./scripts/setup.sh



# Use first regular NVMe disk (non-OC) as non-volatile cache

nvme_disks=$($rootdir/scripts/gen_nvme.sh --json | jq -r \

	   ".config[] | select(.params.traddr != \"$device\").params.traddr")



for disk in $nvme_disks; do

	if has_separate_md $disk; then

		nv_cache=$disk

		break

	fi

done



if [ -z "$nv_cache" ]; then

	# TODO: once CI has devices with separate metadata support fail the test here

	echo "Couldn't find NVMe device to be used as non-volatile cache"

fi



timing_enter ftl

timing_enter bdevperf
@@ -37,11 +54,16 @@ timing_exit bdevperf 


timing_enter restore

run_test suite $testdir/restore.sh $device

if [ -n "$nv_cache" ]; then

	run_test suite $testdir/restore.sh -c $nv_cache $device

fi

timing_exit restore



if [ -n "$nv_cache" ]; then

	timing_enter dirty_shutdown

run_test suite $testdir/dirty_shutdown.sh $device

	run_test suite $testdir/dirty_shutdown.sh -c $nv_cache $device

	timing_exit dirty_shutdown

fi



timing_enter json

run_test suite $testdir/json.sh $device

test/ftl/restore.sh

+22 −6
Original line number Diff line number Diff line
@@ -3,12 +3,23 @@ 
testdir=$(readlink -f $(dirname $0))

rootdir=$(readlink -f $testdir/../..)

source $rootdir/test/common/autotest_common.sh

source $testdir/common.sh



rpc_py=$rootdir/scripts/rpc.py



mount_dir=$(mktemp -d)

pu_start=0

pu_end=3



while getopts ':u:c:' opt; do

	case $opt in

		u) uuid=$OPTARG ;;

		c) nv_cache=$OPTARG ;;

		?) echo "Usage: $0 [-u UUID] [-c NV_CACHE_PCI_BDF] OCSSD_PCI_BDF" && exit 1 ;;

	esac

done

shift $((OPTIND -1))

device=$1

uuid=$2



restore_kill() {

	if mount | grep $mount_dir; then
@@ -30,12 +41,17 @@ $rootdir/app/spdk_tgt/spdk_tgt & svcpid=$! 
# Wait until spdk_tgt starts

waitforlisten $svcpid



if [ -n "$uuid" ]; then

	$rpc_py construct_ftl_bdev -b nvme0 -a $device -l 0-3 -u $uuid

else

	$rpc_py construct_ftl_bdev -b nvme0 -a $device -l 0-3

if [ -n "$nv_cache" ]; then

	nvc_bdev=$(create_nv_cache_bdev nvc0 $device $nv_cache $(($pu_end - $pu_start + 1)))

fi



ftl_construct_args="construct_ftl_bdev -b nvme0 -a $device -l ${pu_start}-${pu_end}"



[ -n "$uuid" ]     && ftl_construct_args+=" -u $uuid"

[ -n "$nv_cache" ] && ftl_construct_args+=" -c $nvc_bdev"



$rpc_py $ftl_construct_args



# Load the nbd driver

modprobe nbd

$rpc_py start_nbd_disk nvme0 /dev/nbd0
@@ -55,7 +71,7 @@ md5sum $mount_dir/testfile > $testdir/testfile.md5 
umount $mount_dir

killprocess $svcpid



$rootdir/app/spdk_tgt/spdk_tgt & svcpid=$!

$rootdir/app/spdk_tgt/spdk_tgt -L ftl_init & svcpid=$!

# Wait until spdk_tgt starts

waitforlisten $svcpid