Commit 4f19ab4d authored by Michal Berger's avatar Michal Berger Committed by Jim Harris
Browse files

scripts/qat_setup: Add support for dh895xcc devices 



These can be found under CYP platform.

To that end, refactor qat_setup.sh so it can support devices based on
their dedicated driver rather than the specific device ID - this will
allow for easier addition of new devices in the future.

Also, configure number of VFs based on total number given ctrl
supports - this is exactly what qat_service is doing while enabling
VFs.

Drop warning about old bug in the uio_pci_generic driver - in
practice we don't hit that under CI since quite a long time.

Slap errexit on top to make sure we exit immediately when writing
to sysfs fails.

Last but not least, run blockdev_crypto_qat test without any top
condition - if qat_setup.sh succeeds, the test should be able to pass.

Signed-off-by: default avatarMichal Berger <michal.berger@intel.com>
Change-Id: I37c4fd319ad7002017f9baf9fdcf3890429aac75
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17086


Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarPawel Piatek <pawelx.piatek@intel.com>
parent 0945b976

autotest.sh

+1 −4
Original line number Diff line number Diff line
@@ -335,11 +335,8 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then 
	if [ $SPDK_TEST_CRYPTO -eq 1 ]; then

		run_test "blockdev_crypto_aesni" $rootdir/test/bdev/blockdev.sh "crypto_aesni"

		run_test "blockdev_crypto_sw" $rootdir/test/bdev/blockdev.sh "crypto_sw"

		# Proceed with the test only if QAT devices are in place

		if [[ $(lspci -d:37c8) ]]; then

		run_test "blockdev_crypto_qat" $rootdir/test/bdev/blockdev.sh "crypto_qat"

	fi

	fi



	if [[ $SPDK_TEST_SCHEDULER -eq 1 ]]; then

		run_test "scheduler" $rootdir/test/scheduler/scheduler.sh

scripts/qat_setup.sh

+34 −32
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#  All rights reserved.

#

shopt -s nullglob

set -e



rootdir=$(readlink -f $(dirname $0))/..

allowed_drivers=("igb_uio" "uio_pci_generic")
@@ -58,9 +59,16 @@ reload_intel_qat() { 


bad_driver=true

driver_to_bind=uio_pci_generic

num_vfs=16



qat_pci_bdfs=($(lspci -Dd:37c8 | awk '{print $1}'))

reload_intel_qat



_qat_pci_bdfs=(

	/sys/bus/pci/drivers/c6xx/0000*

	/sys/bus/pci/drivers/dh895xcc/0000*

)



qat_pci_bdfs=("${_qat_pci_bdfs[@]#*drivers/}")



if [ ${#qat_pci_bdfs[@]} -eq 0 ]; then

	echo "No QAT devices found. Exiting"

	exit 1
@@ -82,45 +90,39 @@ if $bad_driver; then 
	exit 1

fi



reload_intel_qat



# try starting the qat service. If this doesn't work, just treat it as a warning for now.

if ! service qat_service start; then

	echo "failed to start the qat service. Something may be wrong with your 01.org driver."

fi



expected_num_vfs=0 set_num_vfs=0

# configure virtual functions for the QAT cards.

for qat_bdf in "${qat_pci_bdfs[@]}"; do

	if [[ ! -e /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs ]]; then

		echo "($qat_bdf) sriov_numvfs interface missing, is SR-IOV enabled?"

	if [[ ! -e /sys/bus/pci/drivers/$qat_bdf/sriov_numvfs ]]; then

		echo "(${qat_bdf##*/}) sriov_numvfs interface missing, is SR-IOV enabled?"

		continue

	fi

	echo "$num_vfs" > /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs

	num_vfs_set=$(cat /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs)

	num_vfs=$(< "/sys/bus/pci/drivers/$qat_bdf/sriov_totalvfs")

	echo "$num_vfs" > /sys/bus/pci/drivers/$qat_bdf/sriov_numvfs

	num_vfs_set=$(< "/sys/bus/pci/drivers/$qat_bdf/sriov_numvfs")

	if ((num_vfs != num_vfs_set)); then

		echo "Number of VFs set to $num_vfs_set, expected $num_vfs"

	else

		echo "$qat_bdf set to $num_vfs VFs"

		echo "${qat_bdf##*/} set to $num_vfs VFs"

	fi

	((expected_num_vfs += num_vfs))

	((set_num_vfs += num_vfs_set))

done



# Confirm we have all of the virtual functions we asked for.

# Build VF list out of qat_pci_bdfs[@] by slapping /virtfn* to the path to know if we got'em all.

# shellcheck disable=SC2206 # <- intentional

qat_vf_bdfs=(${_qat_pci_bdfs[@]/%//virtfn*})



qat_vf_bdfs=($(lspci -Dd:37c9 | awk '{print $1}'))

if ((${#qat_vf_bdfs[@]} != ${#qat_pci_bdfs[@]} * num_vfs)); then

	echo "Failed to prepare the VFs. Aborting"

if ((expected_num_vfs != set_num_vfs || ${#qat_vf_bdfs[@]} != expected_num_vfs)); then

	echo "Failed to prepare the VFs. Aborting" >&2

	exit 1

fi



# Unbind old driver if necessary.

for vf in "${qat_vf_bdfs[@]}"; do

	old_driver=$(basename $(readlink -f /sys/bus/pci/devices/${vf}/driver))

	if [ $old_driver != "driver" ]; then

		echo "unbinding driver $old_driver from qat VF at BDF $vf"

		echo -n $vf > /sys/bus/pci/drivers/$old_driver/unbind

	fi

done



modprobe uio



# Insert the dpdk uio kernel module.
@@ -136,18 +138,18 @@ else 
	exit 1

fi



echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/remove_id 2> /dev/null

echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/new_id

# Unbind old driver if necessary.

for vf in "${qat_vf_bdfs[@]}"; do

	if ! ls -l /sys/bus/pci/devices/$vf/driver | grep -q $driver_to_bind; then

		echo "unable to bind the driver to the device at bdf $vf"

		if [ "$driver_to_bind" == "uio_pci_generic" ]; then

			echo "Your kernel's uio_pci_generic module does not support binding to virtual functions."

			echo "It likely is missing Linux git commit ID acec09e67 which is needed to bind"

			echo "uio_pci_generic to virtual functions which have no legacy interrupt vector."

			echo "Please build DPDK kernel module for igb_uio and re-run this script specifying the igb_uio driver."

	vf=$(readlink -f "$vf")

	if [[ -e $vf/driver ]]; then

		old_driver=$(basename "$(readlink -f "$vf/driver")")

		if [[ $old_driver != "$driver_to_bind" ]]; then

			echo "unbinding driver $old_driver from qat VF at BDF ${vf##*/}"

			echo "${vf##*/}" > "/sys/bus/pci/drivers/$old_driver/unbind"

		fi

		exit 1

	fi

	echo "$driver_to_bind" > "$vf/driver_override"

	echo "${vf##*/}" > /sys/bus/pci/drivers_probe

done



echo "Properly configured the qat device with driver $driver_to_bind."