Commit c527d09f authored by Krzysztof Karas's avatar Krzysztof Karas Committed by Tomasz Zawadzki
Browse files

hotplug: add new software hotplug test 



Add a new test to sw_hotplug.sh based on SPDK target
in addition to examples/hotplug application.

Change-Id: I352ee4dfe934fa88082101bb92c6db41da10c10c
Signed-off-by: default avatarKrzysztof Karas <krzysztof.karas@intel.com>

Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10088


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent eaebf841

test/nvme/sw_hotplug.sh

+77 −13
Original line number Diff line number Diff line
@@ -11,9 +11,11 @@ cleanup() { 
	kill "$hotplug_pid"

}



# Helper function to remove/attach cotrollers

remove_attach_helper() {

	local hotplug_events=$1

	local hotplug_wait=$2

	local use_bdev=$3

	local dev



	# We need to make sure we wait long enough for hotplug to initialize the devices
@@ -25,10 +27,37 @@ remove_attach_helper() { 
	while ((hotplug_events--)); do

		for dev in "${nvmes[@]}"; do

			echo 1 > "/sys/bus/pci/devices/$dev/remove"

			if $use_bdev; then

				sleep "$hotplug_wait"

				rpc_cmd bdev_get_bdevs \

					| jq '.[] | .driver_specific | .nvme | .[] | .pci_address' \

					| NOT grep $dev

			fi

		done



		echo 1 > "/sys/bus/pci/rescan"



		if $use_bdev; then

			local retries_left=10



			"$rootdir/scripts/setup.sh"



			while [ "$(rpc_cmd bdev_get_bdevs \

				| jq '.[] | .driver_specific | .nvme | .[] | .pci_address' \

				| wc -l)" -lt "$nvme_count" ]; do

				[ $retries_left -gt 0 ] || break

				retries_left=$((retries_left - 1))

				sleep 1

			done



			# Bdev name has already changed, search for the bdev's PCI addresses

			for dev in "${nvmes[@]}"; do

				rpc_cmd bdev_get_bdevs \

					| jq '.[] | .driver_specific | .nvme | .[] | .pci_address' \

					| grep $dev

			done

		fi



		# setup.sh masks failures while writing to sysfs interfaces so let's do

		# this on our own to make sure there's anything for the hotplug to reattach

		# to.
@@ -36,35 +65,70 @@ remove_attach_helper() { 
			echo "$dev" > "/sys/bus/pci/devices/$dev/driver/unbind"

			echo "$dev" > "/sys/bus/pci/drivers/${pci_bus_driver["$dev"]}/bind"

		done



		# Wait now for hotplug to reattach to the devices

		sleep "$hotplug_wait"

	done

}



run_hotplug() {

	"$rootdir/scripts/setup.sh"



	local hotplug_events=3

	local hotplug_wait=6 # This should be enough for more stubborn nvmes in the CI

	local nvmes=($(nvme_in_userspace))

	local nvme_count=${#nvmes[@]}



	xtrace_disable

	cache_pci_bus_sysfs

	xtrace_restore



	trap "cleanup" EXIT



	remove_attach_helper "$hotplug_events" "$hotplug_wait" &

	remove_attach_helper "$hotplug_events" "$hotplug_wait" false &

	hotplug_pid=$!



	"$SPDK_EXAMPLE_DIR/hotplug" \

		-i 0 \

		-t $((hotplug_events * hotplug_wait + hotplug_wait)) \

		-t $((hotplug_events * hotplug_wait + hotplug_wait * 3)) \

		-n $((hotplug_events * nvme_count)) \

		-r $((hotplug_events * nvme_count)) \

		-l warning

}



# SPDK target hotplug

tgt_run_hotplug() {

	local i=0

	local dev



	$SPDK_BIN_DIR/spdk_tgt &

	spdk_tgt_pid=$!



	trap 'killprocess ${spdk_tgt_pid}; echo 1 > /sys/bus/pci/rescan; exit 1' SIGINT SIGTERM EXIT

	waitforlisten $spdk_tgt_pid



	for dev in "${nvmes[@]}"; do

		rpc_cmd bdev_nvme_attach_controller -b "Nvme0$i" -t PCIe -a $dev

		waitforbdev "Nvme0${i}n1" "$hotplug_wait"

		((i = i + 1))

	done



	rpc_cmd bdev_nvme_set_hotplug -e



	remove_attach_helper "$hotplug_events" "$hotplug_wait" true

	# Verify reregistering hotplug poller

	rpc_cmd bdev_nvme_set_hotplug -d

	rpc_cmd bdev_nvme_set_hotplug -e



	remove_attach_helper "$hotplug_events" "$hotplug_wait" true



	trap - SIGINT SIGTERM EXIT

	killprocess $spdk_tgt_pid

}



# Preparation

"$rootdir/scripts/setup.sh"



hotplug_wait=6

hotplug_events=3

nvmes=($(nvme_in_userspace))

nvme_count=${#nvmes[@]}



xtrace_disable

cache_pci_bus_sysfs

xtrace_restore



# Run pci bus hotplug test

run_hotplug



# Run SPDK target based hotplug

tgt_run_hotplug