Commit 2635e73d authored by Michal Berger's avatar Michal Berger Committed by Tomasz Zawadzki
Browse files

scripts/setup: Try to gracefully handle unsupported nic_uio devices 



Under FreeBSD, since we perform a full nic_uio|contigmem reload, we
may affect devices which are not directly supported by the setup.sh.

To mitigate it, try to lookup kernel environment to see if loader
already provides setup for "extra" devices. If so, bail and allow
to continue only when new $FORCE_NIC_UIO_REBIND var is passed.

When $FORCE_NIC_UIO_REBIND is passed, the list of devices to set
via hw.nic_uio.bdfs will include all the "extra" devices to make
sure they are reconfigured as well after the driver's reload -
this includes the 'reset' mode.

Change-Id: I9f1b7784e3cd03c5e7cb4867970e206b6e04bd13
Signed-off-by: default avatarMichal Berger <michal.berger@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24175


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
parent 27d031cd

scripts/common.sh

+1 −1
Original line number Diff line number Diff line
@@ -179,7 +179,7 @@ cache_pci_bus_pciconf() { 


	while read -r pci pci_info; do

		driver=${pci%@*}

		pci=${pci##*pci} pci=${pci%:}

		pci=${pci#*:} pci=${pci%:} # E.g.: nvme0@pci0:0:16:0: -> 0:16:0

		source <(echo "$pci_info")

		# pciconf under FreeBSD 13.1 provides vendor and device IDs in its

		# output under separate, dedicated fields. For 12.x they need to

scripts/setup.sh

+58 −9
Original line number Diff line number Diff line
@@ -93,6 +93,9 @@ function usage() { 
	echo "                  Perform action only against selected type of devices. Supported:"

	echo "                    IOAT|DSA|IAA|VIRTIO|VMD|NVME."

	echo "                  Default is to select all types."

	echo "FORCE_NIC_UIO_REBIND"

	echo "                  When set to 'yes', an attempt to reload nic_uio will be made regardless"

	echo "                  of the kernel environment. Applicable only under FreeBSD."

	exit 0

}
@@ -357,6 +360,8 @@ function collect_devices() { 
		pci_dev_echo "$bdf" "Skipping nvme behind VMD (${nvme_vmd_d["$bdf"]})"

	done



	get_unsupported_nic_uio_hw



	return 0

}
@@ -789,27 +794,59 @@ function status_freebsd() { 
function configure_freebsd_pci() {

	local BDFS



	BDFS+=("${!nvme_d[@]}")

	BDFS+=("${!ioat_d[@]}")

	BDFS+=("${!dsa_d[@]}")

	BDFS+=("${!iaa_d[@]}")

	BDFS+=("${!vmd_d[@]}")

	BDFS+=("$@")



	if ((${#unsupported_nic_uio_hw[@]} > 0)) && [[ $FORCE_NIC_UIO_REBIND != yes ]]; then

		warn_unsupported_nic_uio_hw

		return 1

	fi



	# Drop the domain part from all the addresses

	BDFS=("${BDFS[@]#*:}")

	BDFS+=("${unsupported_nic_uio_hw[@]}")



	if kldstat -n nic_uio &> /dev/null; then

		kldunload nic_uio.ko

	fi



	local IFS=","

	kldunload nic_uio.ko || true

	kenv hw.nic_uio.bdfs="${BDFS[*]}"

	kldload nic_uio.ko

}



function get_unsupported_nic_uio_hw() {

	local bdfs bdf all_devices

	local -g unsupported_nic_uio_hw



	IFS="," read -ra bdfs < <(kenv hw.nic_uio.bdfs 2> /dev/null) || return 0



	for bdf in "${bdfs[@]}"; do

		grep -q "$bdf" <(printf '%s\n' "${!all_devices_d[@]}") || unsupported_nic_uio_hw+=("$bdf")

	done



	return 0

}



function warn_unsupported_nic_uio_hw() {

	cat <<- NIC_UIO



		WARNING: Unsupported devices detected in the nic_uio setup:



		$(printf '  %s\n' "${unsupported_nic_uio_hw[@]}")



		Remove them first or pass FORCE_NIC_UIO_REBIND=yes through the environment.



	NIC_UIO

}



function configure_freebsd() {

	_configure_freebsd "${!nvme_d[@]}" "${!ioat_d[@]}" "${!dsa_d[@]}" "${!iaa_d[@]}" "${!vmd_d[@]}"

}



function _configure_freebsd() {

	if ! check_for_driver_freebsd; then

		echo "DPDK drivers (contigmem and/or nic_uio) are missing, aborting" >&2

		return 1

	fi

	configure_freebsd_pci

	configure_freebsd_pci "$@"

	# If contigmem is already loaded but the HUGEMEM specified doesn't match the

	#  previous value, unload contigmem so that we can reload with the new value.

	if kldstat -q -m contigmem; then
@@ -831,8 +868,20 @@ function configure_freebsd() { 
}



function reset_freebsd() {

	# Don't reap the entire nic_uio setup in case there are unsupported devices in the kernel env

	if ((${#unsupported_nic_uio_hw[@]} > 0)) && [[ $FORCE_NIC_UIO_REBIND != yes ]]; then

		warn_unsupported_nic_uio_hw

		return 1

	fi



	kldunload contigmem.ko || true

	kldunload nic_uio.ko || true



	if ((${#unsupported_nic_uio_hw[@]} > 0)); then

		# HACK: try to be nice and recreate the setup but only with the unsupported devices

		_unsupported_nic_uio_hw=("${unsupported_nic_uio_hw[@]}") unsupported_nic_uio_hw=()

		_configure_freebsd "${_unsupported_nic_uio_hw[@]}"

	fi

}



function set_hp() {