Commit 3920b816 authored by Michal Berger's avatar Michal Berger Committed by Tomasz Zawadzki
Browse files

vm_setup: Make vm_setup.sh to use specific source for repo installations 



Change-Id: I01613720e078a0fa151ed9a4a17e2faf131510bc
Signed-off-by: default avatarMichal Berger <michalx.berger@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2526


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarKarol Latecki <karol.latecki@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
parent 585c9298

test/common/config/vm_setup.sh

+1 −342
Original line number Diff line number Diff line
@@ -41,300 +41,6 @@ else 
	PACKAGEMNG='undefined'

fi



function install_reference_spdk() {

	local last_release

	local output_dir

	local config_params

	local rootdir



	# Create a reference SPDK build for ABI tests

	if echo $CONF | grep -q refspdk; then

		git -C spdk_repo/spdk fetch --tags

		last_release=$(git -C spdk_repo/spdk tag | sort --version-sort | grep -v rc | tail -n1)

		git -C spdk_repo/spdk checkout $last_release

		git -C spdk_repo/spdk submodule update --init

		output_dir="$HOME/spdk_$(tr . _ < <(tr -d '[:alpha:]' <<< $last_release))"



		cp -r spdk_repo/spdk $output_dir



		cat > $HOME/autorun-spdk.conf << EOF

SPDK_BUILD_SHARED_OBJECT=1

SPDK_TEST_AUTOBUILD=1

SPDK_TEST_UNITTEST=1

SPDK_TEST_BLOCKDEV=1

SPDK_TEST_PMDK=1

SPDK_TEST_ISAL=1

SPDK_TEST_REDUCE=1

SPDK_TEST_CRYPTO=1

SPDK_TEST_FTL=1

SPDK_TEST_OCF=1

SPDK_TEST_RAID5=1

SPDK_TEST_RBD=1

SPDK_RUN_ASAN=1

SPDK_RUN_UBSAN=1

EOF



		mkdir -p $HOME/output

		rootdir="$output_dir"

		source $HOME/autorun-spdk.conf

		source $output_dir/test/common/autotest_common.sh



		# Prepare separate, fixed, cmdline for the FreeBSD, Issue #1397.

		if [[ $OSID == freebsd ]]; then

			config_params="--enable-debug --enable-werror"

			config_params+=" --with-idxd --with-fio=/usr/src/fio"

			config_params+=" --disable-unit-tests --without-isal"

			MAKE=gmake

		else

			config_params="$(get_config_params)"

		fi

		$output_dir/configure $(echo $config_params | sed 's/--enable-coverage//g')

		if [[ $OSID != freebsd ]]; then

			$MAKE -C $output_dir $MAKEFLAGS include/spdk/config.h

			CONFIG_OCF_PATH="$output_dir/ocf" $MAKE -C $output_dir/lib/env_ocf $MAKEFLAGS exportlib O=$output_dir/build/ocf.a

			$output_dir/configure $config_params --with-ocf=$output_dir/build/ocf.a --with-shared

		fi

		$MAKE -C $output_dir $MAKEFLAGS

	fi

}



function install_qat() {



	if [ "$PACKAGEMNG" = "dnf" ]; then

		sudo dnf install -y libudev-devel

	elif [ "$PACKAGEMNG" = "apt-get" ]; then

		sudo apt-get install -y libudev-dev

	fi



	if echo $CONF | grep -q qat; then

		qat_tarball=$(basename $DRIVER_LOCATION_QAT)

		kernel_maj=$(uname -r | cut -d'.' -f1)

		kernel_min=$(uname -r | cut -d'.' -f2)



		sudo modprobe -r qat_c62x

		if [ -d /QAT ]; then

			sudo rm -rf /QAT/

		fi



		sudo mkdir /QAT



		wget $DRIVER_LOCATION_QAT

		sudo cp $qat_tarball /QAT/

		(cd /QAT && sudo tar zxof /QAT/$qat_tarball)



		#The driver version 1.7.l.4.3.0-00033 contains a reference to a deprecated function. Remove it so the build won't fail.

		if [ $kernel_maj -le 4 ]; then

			if [ $kernel_min -le 17 ]; then

				sudo sed -i 's/rdtscll(timestamp);/timestamp = rdtsc_ordered();/g' \

					/QAT/quickassist/utilities/osal/src/linux/kernel_space/OsalServices.c || true

			fi

		fi



		(cd /QAT && sudo ./configure --enable-icp-sriov=host && sudo make install)



		if sudo service qat_service start; then

			echo "failed to start the qat service. Something may be wrong with your device or package."

		fi

	fi

}



function install_rocksdb() {

	if echo $CONF | grep -q rocksdb; then

		# Rocksdb is installed for use with the blobfs tests.

		if [ ! -d /usr/src/rocksdb ]; then

			git clone "${GIT_REPO_ROCKSDB}"

			git -C ./rocksdb checkout spdk-v5.6.1

			sudo mv rocksdb /usr/src/

		else

			sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1

			echo "rocksdb already in /usr/src. Not checking out again"

		fi

	fi

}



function install_fio() {

	if echo $CONF | grep -q fio; then

		# This version of fio is installed in /usr/src/fio to enable

		# building the spdk fio plugin.

		local fio_version="fio-3.19"



		if [ ! -d /usr/src/fio ]; then

			if [ ! -d fio ]; then

				git clone "${GIT_REPO_FIO}"

				sudo mv fio /usr/src/

			else

				sudo mv fio /usr/src/

			fi

			(

				git -C /usr/src/fio checkout master \

					&& git -C /usr/src/fio pull \

					&& git -C /usr/src/fio checkout $fio_version \

					&& if [ $OSID == 'freebsd' ]; then

						gmake -C /usr/src/fio -j${jobs} \

							&& sudo gmake -C /usr/src/fio install

					else

						make -C /usr/src/fio -j${jobs} \

							&& sudo make -C /usr/src/fio install

					fi

			)

		else

			echo "fio already in /usr/src/fio. Not installing"

		fi

	fi

}



function install_flamegraph() {

	if echo $CONF | grep -q flamegraph; then

		# Flamegraph is used when printing out timing graphs for the tests.

		if [ ! -d /usr/local/FlameGraph ]; then

			git clone "${GIT_REPO_FLAMEGRAPH}"

			mkdir -p /usr/local

			sudo mv FlameGraph /usr/local/FlameGraph

		else

			echo "flamegraph already installed. Skipping"

		fi

	fi

}



function install_qemu() {

	if echo $CONF | grep -q qemu; then

		# Two versions of QEMU are used in the tests.

		# Stock QEMU is used for vhost. A special fork

		# is used to test OCSSDs. Install both.



		# Packaged QEMU

		if [ "$PACKAGEMNG" = "dnf" ]; then

			sudo dnf install -y qemu-system-x86 qemu-img

		elif [ "$PACKAGEMNG" = "apt-get" ]; then

			sudo apt-get install -y qemu-system-x86 qemu-utils

		elif [ "$PACKAGEMNG" = "pacman" ]; then

			sudo pacman -Sy --needed --noconfirm qemu

		elif [[ $PACKAGEMNG == "yum" ]]; then

			sudo yum install -y qemu-system-x86 qemu-img

		fi



		# Forked QEMU

		SPDK_QEMU_BRANCH=spdk-5.0.0

		mkdir -p qemu

		if [ ! -d "qemu/$SPDK_QEMU_BRANCH" ]; then

			git -C ./qemu clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$SPDK_QEMU_BRANCH"

		else

			echo "qemu already checked out. Skipping"

		fi



		declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH")

		if ((gcc_version >= 9)); then

			# GCC 9 fails to compile Qemu due to some old warnings which were not detected by older versions.

			opt_params+=("--extra-cflags=-Wno-error=stringop-truncation -Wno-error=deprecated-declarations -Wno-error=incompatible-pointer-types -Wno-error=format-truncation")

			opt_params+=("--disable-glusterfs")

		fi



		# Most tsocks proxies rely on a configuration file in /etc/tsocks.conf.

		# If using tsocks, please make sure to complete this config before trying to build qemu.

		if echo $CONF | grep -q tsocks; then

			if hash tsocks 2> /dev/null; then

				opt_params+=("--with-git='tsocks git'")

			fi

		fi



		sed -i s@git://git.qemu.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.gitmodules

		sed -i s@git://git.qemu.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.git/config

		sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.gitmodules

		sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.git/config

		# The qemu configure script places several output files in the CWD.

		(cd qemu/$SPDK_QEMU_BRANCH && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa)



		make -C ./qemu/$SPDK_QEMU_BRANCH -j${jobs}

		sudo make -C ./qemu/$SPDK_QEMU_BRANCH install

	fi

}



function install_vpp() {

	if echo $CONF | grep -q vpp; then

		if [ -d /usr/local/src/vpp ]; then

			echo "vpp already cloned."

			if [ ! -d /usr/local/src/vpp/build-root ]; then

				echo "build-root has not been done"

				echo "remove the $(pwd) and start again"

				exit 1

			fi

		else

			git clone "${GIT_REPO_VPP}"

			git -C ./vpp checkout v19.04.2



			if [ "${OSID}" == 'fedora' ]; then

				if [ ${OSVERSION} -eq 29 ]; then

					git -C ./vpp apply ${VM_SETUP_PATH}/patch/vpp/fedora29-fix.patch

				fi

				if [ ${OSVERSION} -eq 30 ]; then

					git -C ./vpp apply ${VM_SETUP_PATH}/patch/vpp/fedora30-fix.patch

				fi

				if ((OVERSION == 31)); then

					git -C ./vpp apply "$VM_SETUP_PATH/patch/vpp/fedora31-fix.patch"

				fi

			fi



			# vpp depends on python-ply, however some packages on different Fedoras don't

			# provide ply.lex. To make sure vpp won't fail, try to reinstall ply via pip.

			sudo pip3 uninstall -y ply || true

			sudo pip3 install ply || true



			# Installing required dependencies for building VPP

			yes | make -C ./vpp install-dep



			make -C ./vpp build -j${jobs}



			sudo mv ./vpp /usr/local/src/vpp-19.04

		fi

	fi

}



function install_nvmecli() {

	if echo $CONF | grep -q nvmecli; then

		SPDK_NVME_CLI_BRANCH=spdk-1.6

		if [ ! -d nvme-cli ]; then

			git clone "${GIT_REPO_SPDK_NVME_CLI}" -b "$SPDK_NVME_CLI_BRANCH"

		else

			echo "nvme-cli already checked out. Skipping"

		fi

		if [ ! -d "/usr/local/src/nvme-cli" ]; then

			# Changes required for SPDK are already merged on top of

			# nvme-cli, however not released yet.

			# Support for SPDK should be released in nvme-cli >1.11.1

			git clone "https://github.com/linux-nvme/nvme-cli.git" "nvme-cli-cuse"

			git -C ./nvme-cli-cuse checkout "e770466615096a6d41f038a28819b00bc3078e1d"

			make -C ./nvme-cli-cuse

			sudo mv ./nvme-cli-cuse /usr/local/src/nvme-cli

		fi

	fi

}



function install_libiscsi() {

	if echo $CONF | grep -q libiscsi; then

		# We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need

		# to later. Cloning from git is just future proofing the machines.

		if [ ! -d libiscsi ]; then

			git clone "${GIT_REPO_LIBISCSI}"

		else

			echo "libiscsi already checked out. Skipping"

		fi

		(cd libiscsi && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi)

		make -C ./libiscsi -j${jobs}

		sudo make -C ./libiscsi install

	fi

}



function install_git() {

	sudo yum install -y zlib-devel curl-devel

	tar -xzof <(wget -qO- "$GIT_REPO_GIT")

	(cd git-${GIT_VERSION} \

		&& make configure \

		&& ./configure --prefix=/usr/local/git \

		&& sudo make -j${jobs} install)

	sudo sh -c "echo 'export PATH=/usr/local/git/bin:$PATH' >> /etc/bashrc"

	exec $SHELL

}



function usage() {

	echo "This script is intended to automate the environment setup for a linux virtual machine."

	echo "Please run this script as your regular user. The script will make calls to sudo as needed."
@@ -406,38 +112,6 @@ if [ -n "$CONF_PATH" ]; then 
fi



cd ~

GIT_VERSION=2.25.1

: ${GIT_REPO_SPDK=https://github.com/spdk/spdk.git}

export GIT_REPO_SPDK

: ${GIT_REPO_DPDK=https://github.com/spdk/dpdk.git}

export GIT_REPO_DPDK

: ${GIT_REPO_ROCKSDB=https://review.spdk.io/spdk/rocksdb}

export GIT_REPO_ROCKSDB

: ${GIT_REPO_FIO=http://git.kernel.dk/fio.git}

export GIT_REPO_FIO

: ${GIT_REPO_FLAMEGRAPH=https://github.com/brendangregg/FlameGraph.git}

export GIT_REPO_FLAMEGRAPH

: ${GIT_REPO_QEMU=https://github.com/spdk/qemu}

export GIT_REPO_QEMU

: ${GIT_REPO_VPP=https://gerrit.fd.io/r/vpp}

export GIT_REPO_VPP

: ${GIT_REPO_LIBISCSI=https://github.com/sahlberg/libiscsi}

export GIT_REPO_LIBISCSI

: ${GIT_REPO_SPDK_NVME_CLI=https://github.com/spdk/nvme-cli}

export GIT_REPO_SPDK_NVME_CLI

: ${GIT_REPO_INTEL_IPSEC_MB=https://github.com/spdk/intel-ipsec-mb.git}

export GIT_REPO_INTEL_IPSEC_MB

: ${DRIVER_LOCATION_QAT=https://01.org/sites/default/files/downloads//qat1.7.l.4.9.0-00008.tar.gz}

export DRIVER_LOCATION_QAT

: ${GIT_REPO_GIT=https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz}

export GIT_REPO_GIT



if [ $PACKAGEMNG == 'pkg' ]; then

	jobs=$(($(sysctl -n hw.ncpu) * 2))

else

	jobs=$(($(nproc) * 2))

fi



if $UPGRADE; then

	upgrade

fi
@@ -459,22 +133,7 @@ git -C spdk_repo/spdk config submodule.dpdk.url "${GIT_REPO_DPDK}" 
git -C spdk_repo/spdk config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}"

git -C spdk_repo/spdk submodule update --init --recursive



sudo mkdir -p /usr/src



if [ $OSID != 'freebsd' ]; then

	install_libiscsi &

	install_vpp &

	install_nvmecli &

	install_qat &

	install_rocksdb &

	install_flamegraph &

	install_qemu &

fi

install_fio &



wait



install_reference_spdk

source "$vmsetupdir/pkgdep/git"



# create autorun-spdk.conf in home folder. This is sourced by the autotest_common.sh file.

# By setting any one of the values below to 0, you can skip that specific test. If you are