Commit 30116833 authored by Darek Stojaczyk's avatar Darek Stojaczyk Committed by Tomasz Zawadzki
Browse files

autobuild: add SPDK_TEST_NATIVE_DPDK to test again pristine DPDK master 



This was previously done in a Jenkins script. Move it
to autobuild.sh inside the repo where we can all see it.
Use new meson/ninja build system while at it.

To test, use autorun.sh with the following cfg options:
SPDK_RUN_EXTERNAL_DPDK=/tmp/spdk/dpdk/build
SPDK_TEST_NATIVE_DPDK=1

SPDK_RUN_EXTERNAL_DPDK can point to a different path,
but needs to have correct access permissions for current
user,

I had to reorder some code in autobuild.sh. Since
SPDK_RUN_EXTERNAL_DPDK needs to specified the ./configure
script can't be run without existing dpdk directory,
so the dpdk build needs to happen before the initial
./configure as well as the ocf pre-build.

Change-Id: Ibc57094806b361dc3c6acf55942f04a938e5194f
Signed-off-by: default avatarDarek Stojaczyk <dariusz.stojaczyk@intel.com>
Signed-off-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Signed-off-by: default avatarKarol Latecki <karol.latecki@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/868


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 778e4c00

autobuild.sh

+81 −6
Original line number Diff line number Diff line
@@ -28,12 +28,6 @@ cd $rootdir 
# Print some test system info out for the log

date -u

git describe --tags

./configure $config_params

echo "** START ** Info for Hostname: $HOSTNAME"

uname -a

$MAKE cc_version

$MAKE cxx_version

echo "** END ** Info for Hostname: $HOSTNAME"



function ocf_precompile() {

	# We compile OCF sources ourselves
@@ -48,6 +42,76 @@ function ocf_precompile() { 
	./configure $config_params

}



function build_native_dpdk() {

	local external_dpdk_dir

	local external_dpdk_base_dir



	external_dpdk_dir="$SPDK_RUN_EXTERNAL_DPDK"

	external_dpdk_base_dir="$(dirname $external_dpdk_dir)"



	if [[ ! -d "$external_dpdk_base_dir" ]]; then

		sudo mkdir -p "$external_dpdk_base_dir"

		sudo chown -R $(whoami) "$external_dpdk_base_dir"/..

	fi

	orgdir=$PWD



	rm -rf "$external_dpdk_base_dir"

	git clone --branch $SPDK_TEST_NATIVE_DPDK --depth 1 http://dpdk.org/git/dpdk "$external_dpdk_base_dir"

	dpdk_cflags="-fPIC -g -Werror -fcommon"

	dpdk_ldflags=""



	# the drivers we use

	DPDK_DRIVERS=("bus" "bus/pci" "bus/vdev" "mempool/ring")

	# all possible DPDK drivers

	DPDK_ALL_DRIVERS=($(find "$external_dpdk_base_dir/drivers" -mindepth 1 -type d | sed -n "s#^$external_dpdk_base_dir/drivers/##p"))



	if [[ "$SPDK_TEST_CRYPTO" -eq 1 ]]; then

		git clone --branch v0.54 --depth 1 https://github.com/intel/intel-ipsec-mb.git "$external_dpdk_base_dir/intel-ipsec-mb"

		cd "$external_dpdk_base_dir/intel-ipsec-mb"

		$MAKE $MAKEFLAGS all SHARED=n EXTRA_CFLAGS=-fPIC

		DPDK_DRIVERS+=("crypto")

		DPDK_DRIVERS+=("crypto/aesni_mb")

		DPDK_DRIVERS+=("crypto/qat")

		DPDK_DRIVERS+=("compress/qat")

		DPDK_DRIVERS+=("common/qat")

		dpdk_cflags+=" -I$external_dpdk_base_dir/intel-ipsec-mb"

		dpdk_ldflags+=" -L$external_dpdk_base_dir/intel-ipsec-mb"

	fi



	if [[ "$SPDK_TEST_REDUCE" -eq 1 ]]; then

		git clone --branch v2.29.0 --depth 1 https://github.com/intel/isa-l.git "$external_dpdk_base_dir/isa-l"

		cd "$external_dpdk_base_dir/isa-l"

		./autogen.sh

		./configure CFLAGS="-fPIC -g -O2" --enable-shared=no

		ln -s $PWD/include $PWD/isa-l

		$MAKE $MAKEFLAGS all

		DPDK_DRIVERS+=("compress")

		DPDK_DRIVERS+=("compress/isal")

		DPDK_DRIVERS+=("compress/qat")

		DPDK_DRIVERS+=("common/qat")

		dpdk_cflags+=" -I$external_dpdk_base_dir/isa-l"

		dpdk_ldflags+=" -L$external_dpdk_base_dir/isa-l/.libs"

	fi



	# Use difference between DPDK_ALL_DRIVERS and DPDK_DRIVERS as a set of DPDK drivers we don't want or

	# don't need to build.

	DPDK_DISABLED_DRIVERS=($(sort <(printf "%s\n" "${DPDK_DRIVERS[@]}") <(printf "%s\n" "${DPDK_ALL_DRIVERS[@]}") | uniq -u))



	cd $external_dpdk_base_dir

	if [ "$(uname -s)" = "Linux" ]; then

		dpdk_cflags+=" -Wno-stringop-overflow"

	fi



	meson build-tmp --prefix="$external_dpdk_dir" --libdir lib \

		-Denable_docs=false -Denable_kmods=false -Dtests=false \

		-Dc_args="$dpdk_cflags $dpdk_ldflags" \

		-Dmachine=native -Ddisable_drivers=$(printf "%s," "${DPDK_DISABLED_DRIVERS[@]}")

	ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS

	ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS install



	cd "$orgdir"

}



function make_fail_cleanup() {

	if [ -d $out/scan-build-tmp ]; then

		scanoutput=$(ls -1 $out/scan-build-tmp/)
@@ -177,6 +241,17 @@ if [ $SPDK_RUN_UBSAN -eq 1 ]; then 
	run_test "ubsan" echo "using ubsan"

fi



if [ -n "$SPDK_TEST_NATIVE_DPDK" ]; then

	run_test "build_native_dpdk" build_native_dpdk

fi



./configure $config_params

echo "** START ** Info for Hostname: $HOSTNAME"

uname -a

$MAKE cc_version

$MAKE cxx_version

echo "** END ** Info for Hostname: $HOSTNAME"



if [ "$SPDK_TEST_AUTOBUILD" -eq 1 ]; then

	run_test "autobuild" autobuild_test_suite $1

else

autotest.sh

+3 −5
Original line number Diff line number Diff line
@@ -9,15 +9,13 @@ if [[ ! -f $1 ]]; then 
	exit 1

fi



# always test with SPDK shared objects.

export SPDK_LIB_DIR="$rootdir/build/lib"



source "$1"

source "$rootdir/test/common/autotest_common.sh"

source "$rootdir/test/nvmf/common.sh"



# always test with SPDK shared objects.

export SPDK_LIB_DIR="$rootdir/build/lib"

export DPDK_LIB_DIR="$rootdir/dpdk/build/lib"

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR



if [ $EUID -ne 0 ]; then

	echo "$0 must be run as root"

	exit 1

test/common/autotest_common.sh

+5 −0
Original line number Diff line number Diff line
@@ -120,6 +120,8 @@ export SPDK_TEST_FTL_EXTENDED 
export SPDK_TEST_VMD

: ${SPDK_TEST_OPAL=0}

export SPDK_TEST_OPAL

: ${SPDK_TEST_NATIVE_DPDK}

export SPDK_TEST_NATIVE_DPDK

: ${SPDK_AUTOTEST_X=true}

export SPDK_AUTOTEST_X

: ${SPDK_TEST_RAID5=0}
@@ -127,6 +129,9 @@ export SPDK_TEST_RAID5 
: ${SPDK_TEST_URING=0}

export SPDK_TEST_URING



export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib"

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR



# Tell setup.sh to wait for block devices upon each reset

export PCI_BLOCK_SYNC_ON_RESET=yes

test/common/skipped_tests.txt

+1 −0
Original line number Diff line number Diff line
@@ -39,6 +39,7 @@ vmd_perf 
bdev_fio_rw_verify_ext

bdev_fio_trim_ext

bdev_reset

build_native_dpdk

iscsi_tgt_digest

iscsi_tgt_data_digest

iscsi_tgt_pmem