Commit 765f4a73 authored by Seth Howell's avatar Seth Howell Committed by Jim Harris
Browse files

test/nvmf: add initiator timeout test. 



This test is meant to simulate uncommon network situations where lots of
I/O are getting dropped before they get to the initiator. Essentially,
we are trying to trigger the initiator timeout in the kernel which
disconnects from the target and reconnects after 10 seconds.

This test is different from the other tests that we have currently
because it doesn't rely on killing either the target or initiator.
Instead it expects the target to stay up and properly respond to the
initiator error conditions. The hope is that the fio job running on the
initiator can still complete successfully and the target doesn't crash
due to improperly handling the disconnect and reconnect.

This test should be added to the nightly CI jobs because it is rather
time consuming.

Change-Id: Id457d9eb21a980140f065663547b89b2c69ace93
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/464459


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarBroadcom SPDK FC-NVMe CI <spdk-ci.pdl@broadcom.com>
parent d2424824

test/nvmf/nvmf.sh

+1 −0
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ run_test suite test/nvmf/target/create_transport.sh $TEST_ARGS 
if [ $RUN_NIGHTLY -eq 1 ]; then

	run_test suite test/nvmf/target/fuzz.sh $TEST_ARGS

	run_test suite test/nvmf/target/multiconnection.sh $TEST_ARGS

	run_test suite test/nvmf/target/initiator_timeout.sh $TEST_ARGS

fi



run_test suite test/nvmf/target/nmic.sh $TEST_ARGS

test/nvmf/target/initiator_timeout.sh

0 → 100755
+73 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash



testdir=$(readlink -f $(dirname $0))

rootdir=$(readlink -f $testdir/../../..)

source $rootdir/test/common/autotest_common.sh

source $rootdir/test/nvmf/common.sh



MALLOC_BDEV_SIZE=64

MALLOC_BLOCK_SIZE=512



rpc_py="$rootdir/scripts/rpc.py"



timing_enter initiator_timeout

nvmftestinit

nvmfappstart "-m 0xF"



trap "process_shm --id $NVMF_APP_SHM_ID; killprocess $nvmfpid; nvmftestfini $1; exit 1" SIGINT SIGTERM EXIT



$rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0



# We cannot configure the bdev with an incredibly high latency up front because connect will not work properly.

$rpc_py bdev_delay_create -b Malloc0 -d Delay0 -r 30 -t 30 -w 30 -n 30



$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192

$rpc_py nvmf_subsystem_create nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001

$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Delay0

$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT



nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT"



waitforblk "nvme0n1"



# Once our timed out I/O complete, we will still have 10 sec of I/O.

$rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t write -r 60 -v &

fio_pid=$!



sleep 3



# The kernel initiator has a default timeout of 30 seconds. delay for 31 to trigger initiator reconnect.

$rpc_py bdev_delay_update_latency Delay0 avg_read 31000000

$rpc_py bdev_delay_update_latency Delay0 avg_write 31000000

$rpc_py bdev_delay_update_latency Delay0 p99_read 31000000

$rpc_py bdev_delay_update_latency Delay0 p99_write 310000000



sleep 3



# Reset these values so that subsequent I/O will complete in a timely manner.

$rpc_py bdev_delay_update_latency Delay0 avg_read 30

$rpc_py bdev_delay_update_latency Delay0 avg_write 30

$rpc_py bdev_delay_update_latency Delay0 p99_read 30

$rpc_py bdev_delay_update_latency Delay0 p99_write 30



fio_status=0

wait $fio_pid || fio_status=$?



nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" || true



if [ $fio_status -eq 0 ]; then

        echo "nvmf hotplug test: fio successful as expected"

else

        echo "nvmf hotplug test: fio failed, expected success"

        nvmftestfini

        exit 1

fi



$rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1



rm -f ./local-job0-0-verify.state



trap - SIGINT SIGTERM EXIT



nvmftestfini

timing_exit initiator_timeout