Commit 91751e1e authored by Monica Kenguva's avatar Monica Kenguva Committed by Tomasz Zawadzki
Browse files

test/nvmf: Multipath testing with ANA states support 



Signed-off-by: default avatarMonica Kenguva <monica.kenguva@intel.com>
Change-Id: Ide3af4fd027450eeac523e7078ba41ab0572ed80
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10197


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
parent 67357518

scripts/vagrant/autorun-spdk.conf

+1 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ SPDK_TEST_VHOST=0 
SPDK_TEST_VHOST_INIT=0

SPDK_TEST_BLOCKDEV=1

SPDK_TEST_URING=0

SPDK_TEST_USDT=0

# doesn't work on vm

SPDK_TEST_IOAT=0

SPDK_TEST_BLOBFS=0

test/common/autotest_common.sh

+6 −0
Original line number Diff line number Diff line
@@ -151,6 +151,8 @@ export SPDK_AUTOTEST_X 
export SPDK_TEST_RAID5

: ${SPDK_TEST_URING=0}

export SPDK_TEST_URING

: ${SPDK_TEST_USDT=0}

export SPDK_TEST_USDT

: ${SPDK_TEST_USE_IGB_UIO:=0}

export SPDK_TEST_USE_IGB_UIO

: ${SPDK_TEST_SCHEDULER:=0}
@@ -370,6 +372,10 @@ function get_config_params() { 
		config_params+=' --with-rdma'

	fi



	if [ $SPDK_TEST_USDT -eq 1 ]; then

		config_params+=" --with-usdt"

	fi



	if [ $(uname -s) == "FreeBSD" ]; then

		intel="hw.model: Intel"

		cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15)

test/nvmf/host/multipath.sh

0 → 100755
+114 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash



testdir=$(readlink -f $(dirname $0))

rootdir=$(readlink -f $testdir/../../..)

source $rootdir/test/common/autotest_common.sh

source $rootdir/test/nvmf/common.sh



MALLOC_BDEV_SIZE=64

MALLOC_BLOCK_SIZE=512



rpc_py="$rootdir/scripts/rpc.py"

bpf_sh="$rootdir/scripts/bpftrace.sh"



bdevperf_rpc_sock=/var/tmp/bdevperf.sock



# NQN prefix to use for subsystem NQNs

NQN=nqn.2016-06.io.spdk:cnode1



nvmftestinit



nvmfappstart -m 0x3

nvmfapp_pid=$!



$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192

$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0

# Set -r to enable ANA reporting feature

$rpc_py nvmf_create_subsystem $NQN -a -s SPDK00000000000001 -r -m 2

$rpc_py nvmf_subsystem_add_ns $NQN Malloc0

$rpc_py nvmf_subsystem_add_listener $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT

$rpc_py nvmf_subsystem_add_listener $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT



$rootdir/test/bdev/bdevperf/bdevperf -m 0x4 -z -r $bdevperf_rpc_sock -q 128 -o 4096 -w verify -t 90 &> $testdir/try.txt &

bdevperf_pid=$!



trap 'process_shm --id $NVMF_APP_SHM_ID; rm -f $testdir/try.txt; killprocess $bdevperf_pid; nvmftestfini; exit 1' SIGINT SIGTERM EXIT

waitforlisten $bdevperf_pid $bdevperf_rpc_sock



# Create a controller and set multipath behavior

# bdev_retry_count is set to -1 means infinite reconnects

$rpc_py -s $bdevperf_rpc_sock bdev_nvme_set_options -r -1

# -l -1 ctrlr_loss_timeout_sec -1 means infinite reconnects

# -o 10 reconnect_delay_sec time to delay a reconnect retry is limited to 10 sec

$rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b Nvme0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT -f ipv4 -n $NQN -l -1 -o 10

$rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b Nvme0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT -f ipv4 -n $NQN -x multipath -l -1 -o 10



function set_ANA_state() {

	$rpc_py nvmf_subsystem_listener_set_ana_state $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT -n $1

	$rpc_py nvmf_subsystem_listener_set_ana_state $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT -n $2

}



# check for io on the expected ANA state port

function confirm_io_on_port() {

	$bpf_sh $nvmfapp_pid $rootdir/scripts/bpf/nvmf_path.bt &> $testdir/trace.txt &

	dtrace_pid=$!

	sleep 6

	active_port=$($rpc_py nvmf_subsystem_get_listeners $NQN | jq -r '.[] | select (.ana_states[0].ana_state=="'$1'") | .address.trsvcid')

	cat $testdir/trace.txt

	port=$(cut < $testdir/trace.txt -d ']' -f1 | awk '$1=="@path[10.0.0.2," {print $2}' | sed -n '1p')

	[[ "$active_port" == "$port" ]]

	[[ "$port" == "$2" ]]

	kill $dtrace_pid

	rm -f $testdir/trace.txt

}



$rootdir/test/bdev/bdevperf/bdevperf.py -t 120 -s $bdevperf_rpc_sock perform_tests &

rpc_pid=$!



sleep 1



# Set ANA state to each listener

set_ANA_state non_optimized optimized

# Check the IO on expected port with ANA state set

confirm_io_on_port "optimized" $NVMF_SECOND_PORT



# Check traffic paths with different ANA states

set_ANA_state non_optimized inaccessible

confirm_io_on_port "non_optimized" $NVMF_PORT



set_ANA_state inaccessible optimized

confirm_io_on_port "optimized" $NVMF_SECOND_PORT



# Not expecting the io on any port

set_ANA_state inaccessible inaccessible

confirm_io_on_port "" ""



set_ANA_state non_optimized optimized

confirm_io_on_port "optimized" $NVMF_SECOND_PORT



# Remove listener to monitor multipath function

$rpc_py nvmf_subsystem_remove_listener $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT

sleep 1



# Expect IO on alternate path

confirm_io_on_port "non_optimized" $NVMF_PORT



# Add listener back with optimized state, traffic should switch to optimized port

$rpc_py nvmf_subsystem_add_listener $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT

$rpc_py nvmf_subsystem_listener_set_ana_state $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT -n optimized



# wait for the io to switch to second port

sleep 2

confirm_io_on_port "optimized" $NVMF_SECOND_PORT



wait $rpc_pid

cat $testdir/try.txt



killprocess $bdevperf_pid



$rpc_py nvmf_delete_subsystem $NQN



trap - SIGINT SIGTERM EXIT



rm -f $testdir/try.txt

nvmftestfini

test/nvmf/nvmf.sh

+3 −0
Original line number Diff line number Diff line
@@ -72,6 +72,9 @@ timing_enter host 
run_test "nvmf_identify" test/nvmf/host/identify.sh "${TEST_ARGS[@]}"

run_test "nvmf_perf" test/nvmf/host/perf.sh "${TEST_ARGS[@]}"

run_test "nvmf_failover" test/nvmf/host/failover.sh "${TEST_ARGS[@]}"

if [[ $SPDK_TEST_USDT -eq 1 ]]; then

	run_test "nvmf_multipath" test/nvmf/host/multipath.sh "${TEST_ARGS[@]}"

fi

run_test "nvmf_discovery" test/nvmf/host/discovery.sh "${TEST_ARGS[@]}"



# TODO: disabled due to intermittent failures (RDMA_CM_EVENT_UNREACHABLE/ETIMEDOUT)