Commit db0d8682 authored by Michal Berger's avatar Michal Berger Committed by Tomasz Zawadzki
Browse files

scripts/setup: Attempt to unbind all devices from target iommu group 



Depending on the pci topology, there may be setups where target nvme
is part of iommu group which consists of more devices bound to
various different drivers. In such a case, DPDK will refuse to
configure the device and mark given iommu group as "not viable".

To workaround this, we may try to unbind all remaining devices from
their drivers (binding to vfio or uio may fail, depending on what
kind of device we are dealing with). However, since this is an
invasive operation it should be done only on demand.

Successfully tested under WFP platform where nvme was connected to
fanout pcie switches, however, results may vary across the systems.

Change-Id: I4f315cf2e4178841859ecbba5ca09cae5fcbd497
Signed-off-by: default avatarMichal Berger <michal.berger@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/18357


Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarKarol Latecki <karol.latecki@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 4749be24

scripts/setup.sh

+34 −3
Original line number Diff line number Diff line
@@ -85,6 +85,9 @@ function usage() { 
	echo "PCI_BLOCK_SYNC_ON_RESET"

	echo "                  If set in the environment, the attempt to wait for block devices associated"

	echo "                  with given PCI device will be made upon reset"

	echo "UNBIND_ENTIRE_IOMMU_GROUP"

	echo "                  If set, all devices from nvme's iommu group will be unbound from their drivers."

	echo "                  Use with caution."

	exit 0

}
@@ -130,9 +133,9 @@ function pci_dev_echo() { 
	echo "$bdf (${pci_ids_vendor["$bdf"]#0x} ${pci_ids_device["$bdf"]#0x}): $*"

}



function linux_bind_driver() {

	bdf="$1"

	driver_name="$2"

function probe_driver() {

	local bdf=$1

	local driver_name=$2

	old_driver_name=${drivers_d["$bdf"]:-no driver}



	if [[ $driver_name == "$old_driver_name" ]]; then
@@ -163,6 +166,13 @@ function linux_bind_driver() { 
		pci_dev_echo "$bdf" "failed to bind to $driver_name, aborting"

		return 1

	fi

}



function linux_bind_driver() {

	local bdf="$1"

	local driver_name="$2"



	probe_driver "$bdf" "$driver_name"



	iommu_group=$(basename $(readlink -f /sys/bus/pci/devices/$bdf/iommu_group))

	if [ -e "/dev/vfio/$iommu_group" ]; then
@@ -170,6 +180,27 @@ function linux_bind_driver() { 
			chown "$TARGET_USER" "/dev/vfio/$iommu_group"

		fi

	fi



	local iommug=("/sys/bus/pci/devices/$bdf/iommu_group/devices/"!($bdf))

	local _bdf _driver

	if ((${#iommug[@]} > 0)) && [[ $driver_name == vfio* ]]; then

		pci_dev_echo "$bdf" "WARNING: detected multiple devices (${#iommug[@]}) under the same IOMMU group!"

		for _bdf in "${iommug[@]}"; do

			_driver=$(readlink -f "$_bdf/driver")

			if [[ ! -e $_driver || ${_driver##*/} == "$driver_name" ]]; then

				continue

			fi

			# See what DPDK considers to be a "viable" iommu group: dpdk/lib/eal/linux/eal_vfio.c -> rte_vfio_setup_device()

			pci_dev_echo "$bdf" "WARNING: ${_bdf##*/} not bound to $driver_name (${_driver##*/})"

			pci_dev_echo "$bdf" "WARNING All devices in the IOMMU group must be bound to the same driver or unbound"

			if [[ $UNBIND_ENTIRE_IOMMU_GROUP == yes ]]; then

				pci_dev_echo "$bdf" "WARNING: Attempting to unbind ${_bdf##*/}"

				drivers_d["${_bdf##*/}"]=${_driver##*/}

				probe_driver "${_bdf##*/}" none

			fi

		done

	fi



}



function linux_unbind_driver() {

test/common/autotest_common.sh

+3 −0
Original line number Diff line number Diff line
@@ -255,6 +255,9 @@ export VFIO_QEMU_BIN=${VFIO_QEMU_BIN:-} 


export AR_TOOL=$rootdir/scripts/ar-xnvme-fixer



# For testing nvmes which are attached to some sort of a fanout switch in the CI pool

export UNBIND_ENTIRE_IOMMU_GROUP=${UNBIND_ENTIRE_IOMMU_GROUP:-no}



# pass our valgrind desire on to unittest.sh

if [ $SPDK_RUN_VALGRIND -eq 0 ]; then

	export valgrind=''