Commit 3c60910e authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Tomasz Zawadzki
Browse files

test/sma: add discovery test 



The test checks that it's possible to attach remote volumes through
discovery service.  The tests starts up three SPDK instances: one is being
managed by SMA, while the other two are acting as remote storage nodes.
The test verifies that volumes can be successfully discvered and
attached to a device and that the connections to the discovery
subsystems are refcounted correctly.

Signed-off-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I3488b822880b99ef9cfd70e03de3ed9054c13901
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12414


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
parent 0c2b10f2

test/common/skipped_tests.txt

+1 −0
Original line number Diff line number Diff line
@@ -93,3 +93,4 @@ zoned_fio 
# SMA tests - disabled in CI for now

sma_nvmf_tcp

sma_plugins

sma_discovery

test/sma/common.sh

+7 −0
Original line number Diff line number Diff line
@@ -10,3 +10,10 @@ function sma_waitforlisten() { 
	done

	return 1

}



function uuid2base64() {

	python <<- EOF

		import base64, uuid

		print(base64.b64encode(uuid.UUID("$1").bytes).decode())

	EOF

}

test/sma/discovery.sh

0 → 100755
+404 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash



testdir=$(readlink -f "$(dirname "$0")")

rootdir=$(readlink -f "$testdir/../..")



source "$rootdir/test/common/autotest_common.sh"

source "$testdir/common.sh"



rpc_py="$rootdir/scripts/rpc.py"

sma_py="$rootdir/scripts/sma-client.py"



t1sock='/var/tmp/spdk.sock1'

t2sock='/var/tmp/spdk.sock2'

invalid_port=8008

t1dscport=8009

t2dscport1=8010

t2dscport2=8011

t1nqn='nqn.2016-06.io.spdk:node1'

t2nqn='nqn.2016-06.io.spdk:node2'

hostnqn='nqn.2016-06.io.spdk:host0'



function cleanup() {

	killprocess $smapid

	killprocess $tgtpid

	killprocess $t1pid

	killprocess $t2pid

}



function format_endpoints() {

	local eps=("$@")

	for ((i = 0; i < ${#eps[@]}; i++)); do

		cat <<- EOF

			{

				"trtype": "tcp",

				"traddr": "127.0.0.1",

				"trsvcid": "${eps[i]}"

			}

		EOF

		if ! ((i + 1 == ${#@})); then

			echo ,

		fi

	done

}



function format_volume() {

	local volume_id=$1

	shift



	cat <<- EOF

		"volume": {

			"volume_id": "$(uuid2base64 $volume_id)",

			"nvmf": {

				"hostnqn": "$hostnqn",

				"discovery": {

					"discovery_endpoints": [

						$(format_endpoints "$@")

					]

				}

			}

		}

	EOF

}



function create_device() {

	local nqn=$1

	local volume_id=$2

	local volume=



	shift

	if [[ -n "$volume_id" ]]; then

		volume="$(format_volume "$@"),"

	fi



	$sma_py <<- EOF

		{

			"method": "CreateDevice",

			"params": {

				$volume

				"nvmf_tcp": {

					"subnqn": "$nqn",

					"adrfam": "ipv4",

					"traddr": "127.0.0.1",

					"trsvcid": "4419"

				}

			}

		}

	EOF

}



function delete_device() {

	$sma_py <<- EOF

		{

			"method": "DeleteDevice",

			"params": {

				"handle": "$1"

			}

		}

	EOF

}



function attach_volume() {

	local device_id=$1



	shift

	$sma_py <<- EOF

		{

			"method": "AttachVolume",

			"params": {

				$(format_volume "$@"),

				"device_handle": "$device_id"

			}

		}

	EOF

}



function detach_volume() {

	$sma_py <<- EOF

		{

			"method": "DetachVolume",

			"params": {

				"device_handle": "$1",

				"volume_id": "$(uuid2base64 $2)"

			}

		}

	EOF

}



trap "cleanup; exit 1" SIGINT SIGTERM EXIT



# Start two remote targets

$rootdir/build/bin/spdk_tgt -r $t1sock &

t1pid=$!

$rootdir/build/bin/spdk_tgt -r $t2sock &

t2pid=$!



# One target that the SMA will configure

$rootdir/build/bin/spdk_tgt &

tgtpid=$!



# And finally the SMA itself

$rootdir/scripts/sma.py -c <(

	cat <<- EOF

		discovery_timeout: 5

		devices:

		  - name: 'nvmf_tcp'

	EOF

) &

smapid=$!



waitforlisten $t1pid

waitforlisten $t2pid



# Prepare the targets.  The first one has a single subsystem with a single volume and a single

# discovery listener.  The second one also has a single subsystem, but has two volumes attached to

# it and has two discovery listeners.

t1uuid=$(uuidgen)

t2uuid=$(uuidgen)

t2uuid2=$(uuidgen)



$rpc_py -s $t1sock <<- EOF

	nvmf_create_transport -t tcp

	bdev_null_create null0 128 4096 -u $t1uuid

	nvmf_create_subsystem $t1nqn -s SPDK00000000000001 -d SPDK_Controller1

	nvmf_subsystem_add_host $t1nqn $hostnqn

	nvmf_subsystem_add_ns $t1nqn $t1uuid

	nvmf_subsystem_add_listener $t1nqn -t tcp -a 127.0.0.1 -s 4420

	nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t1dscport

EOF



$rpc_py -s $t2sock <<- EOF

	nvmf_create_transport -t tcp

	bdev_null_create null0 128 4096 -u $t2uuid

	bdev_null_create null1 128 4096 -u $t2uuid2

	nvmf_create_subsystem $t2nqn -s SPDK00000000000001 -d SPDK_Controller1

	nvmf_subsystem_add_host $t2nqn $hostnqn

	nvmf_subsystem_add_ns $t2nqn $t2uuid

	nvmf_subsystem_add_ns $t2nqn $t2uuid2

	nvmf_subsystem_add_listener $t2nqn -t tcp -a 127.0.0.1 -s 4421

	nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t2dscport1

	nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t2dscport2

EOF



# Wait until the SMA starts listening

sma_waitforlisten



localnqn='nqn.2016-06.io.spdk:local0'



# Create a device

device_id=$(create_device $localnqn | jq -r '.handle')



# Check that it's been created

$rpc_py nvmf_get_subsystems $localnqn



# Attach a volume specifying both targets

attach_volume $device_id $t1uuid $t1dscport $t2dscport1



# Check that a connection has been made to discovery services on both targets

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]]



$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1



# Check that the volume was attached to the device

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]



# Attach the other volume, this time specify only single target

attach_volume $device_id $t2uuid $t2dscport1



# Check that both volumes are attached to the device

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]]

$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t1uuid

$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid



# Detach the first volume

detach_volume $device_id $t1uuid



# Check that there's a connection to a single target now (because we've only specified a single

# target when connecting the other volume).

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1

# Check that the volume was actually removed

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t2uuid" ]]



# Detach the other volume

detach_volume $device_id $t2uuid



# And verify it's gone too

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]]



# Check that specifying an invalid volume UUID results in an error

NOT attach_volume $device_id $(uuidgen) $t1dscport

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]]



# Attach them again, this time both volumes specify both targets

volumes=($t1uuid $t2uuid)

for volume_id in "${volumes[@]}"; do

	attach_volume $device_id $volume_id $t1dscport $t2dscport1

done



[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]]

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1

$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t1uuid

$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid



# Detach one and see that both targets are still connected

detach_volume $device_id $t1uuid



[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]]

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1



# Delete the device and verify that this also causes the volumes to be disconnected

delete_device $device_id



[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]

NOT $rpc_py nvmf_get_subsystems $localnqn



# Create a device and attach a volume immediately

device_id=$(create_device $localnqn $t1uuid $t1dscport | jq -r '.handle')



# Verify that there's a connection to the target and the volume is attached to the device

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]



# Make sure it's also possible to detach it

detach_volume $device_id $t1uuid



[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]]



# Check that discovery referrals work correctly

attach_volume $device_id $t2uuid $t2dscport1 $t2dscport2



# Check that only a single connection has been made

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t2uuid" ]]



# Add the other volume from the same target/subsystem, but use a single discovery endpoint

attach_volume $device_id $t2uuid2 $t2dscport2



# Check that the volume was attached to the subsystem, but no extra connection has been made

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]]

$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid

$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid2



# Reset the device

delete_device $device_id

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]



device_id=$(create_device $localnqn | jq -r '.handle')



# Check subsystem NQN verification, start with a valid one

$sma_py <<- EOF

	{

		"method": "AttachVolume",

		"params": {

			"volume": {

				"volume_id": "$(uuid2base64 $t1uuid)",

				"nvmf": {

					"hostnqn": "$hostnqn",

					"subnqn": "$t1nqn",

					"discovery": {

						"discovery_endpoints": [

							{

								"trtype": "tcp",

								"traddr": "127.0.0.1",

								"trsvcid": "$t1dscport"

							}

						]

					}

				}

			},

			"device_handle": "$device_id"

		}

	}

EOF



[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]



# Then check incorrect subnqn

NOT $sma_py <<- EOF

	{

		"method": "AttachVolume",

		"params": {

			"volume": {

				"volume_id": "$(uuid2base64 $t2uuid)",

				"nvmf": {

					"hostnqn": "$hostnqn",

					"subnqn": "${t2nqn}-invalid",

					"discovery": {

						"discovery_endpoints": [

							{

								"trtype": "tcp",

								"traddr": "127.0.0.1",

								"trsvcid": "$t2dscport1"

							}

						]

					}

				}

			},

			"device_handle": "$device_id"

		}

	}

EOF



# Verify the volume hasn't been attached

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]



# Check incorrect hostnqn

NOT $sma_py <<- EOF

	{

		"method": "AttachVolume",

		"params": {

			"volume": {

				"volume_id": "$(uuid2base64 $t2uuid)",

				"nvmf": {

					"hostnqn": "${hostnqn}-invalid",

					"discovery": {

						"discovery_endpoints": [

							{

								"trtype": "tcp",

								"traddr": "127.0.0.1",

								"trsvcid": "$t2dscport1"

							}

						]

					}

				}

			},

			"device_handle": "$device_id"

		}

	}

EOF



# Verify that the volume wasn't attached

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]

[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]



# Check that the the attach will fail if there's nobody listening on the discovery endpoint

NOT attach_volume $device_id $(uuidgen) $invalid_port

[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]

$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport



delete_device $device_id



cleanup

trap - SIGINT SIGTERM EXIT

test/sma/nvmf_tcp.sh

+0 −7
Original line number Diff line number Diff line
@@ -11,13 +11,6 @@ function cleanup() { 
	killprocess $smapid

}



function uuid2base64() {

	python <<- EOF

		import base64, uuid

		print(base64.b64encode(uuid.UUID("$1").bytes).decode())

	EOF

}



function create_device() {

	"$rootdir/scripts/sma-client.py" <<- EOF

		{

test/sma/sma.sh

+1 −0
Original line number Diff line number Diff line
@@ -7,3 +7,4 @@ source "$rootdir/test/common/autotest_common.sh" 


run_test "sma_nvmf_tcp" $testdir/nvmf_tcp.sh

run_test "sma_plugins" $testdir/plugins.sh

run_test "sma_discovery" $testdir/discovery.sh