Commit 51e03e60 authored by Michal Berger's avatar Michal Berger Committed by Tomasz Zawadzki
Browse files

pkgdep/git: Add support for installing DPDK's contigmem driver 



The main goal here is to arm freebsd environment with ability to
load aforementioned driver early on during kernel's bootup. This
should mitigate potential issues with allocating enough
contiguous memory space for DPDK during tests.

dpdk-stable's latest branch is used by default as a source for
the driver.

As we are at it, small reshuffling of the freebsd_*() functions
is done to make sure it's clear what they are responsible for.

CI autorun workflows should utilize this driver under FreeBSD in
the following manner:

  # During qcow2 build
  $ sudo  ./autotest_setup.sh  [...] --test-conf="contigmem"

The above will build, install and prep the driver for the loader -
default of 2GB will be requested on boot which should be enough
and match default requirement by autotest.

  # during tests under FreeBSD
  export SKIP_HUGE=yes USE_INSTALLED_CONTIGMEM=yes
  ...

The SKIP_HUGE flag is meant for setup.sh to avoid any tweaking
of contigmem buffers; USE_INSTALLED_CONTIGMEM makes sure that
pre-installed driver won't be overwritten by autobuild's.

Change-Id: I16216d9b17144d046cbcffa9d66783d9b8e91102
Signed-off-by: default avatarMichal Berger <michal.berger@nutanix.com>
Reviewed-on: https://review.spdk.io/c/spdk/spdk/+/26376


Tested-by: default avatarSPDK Automated Test System <spdkbot@gmail.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz@tzawadzki.com>
Reviewed-by: default avatarKonrad Sztyber <ksztyber@nvidia.com>
Community-CI: Mellanox Build Bot
parent 4cbbcdc8

autotest.sh

+1 −1
Original line number Diff line number Diff line
@@ -63,7 +63,7 @@ src=$(readlink -f $(dirname $0)) 
out=$output_dir

cd $src



freebsd_update_contigmem_mod

freebsd_update_mods

freebsd_set_maxsock_buf



if [[ $CONFIG_COVERAGE == y ]]; then

scripts/setup.sh

+2 −1
Original line number Diff line number Diff line
@@ -849,7 +849,8 @@ function reset_freebsd() { 
		return 1

	fi



	kldunload contigmem.ko || true

	# Don't touch contigmem if requested

	[[ $SKIP_HUGE == yes ]] || kldunload contigmem.ko || true

	kldunload nic_uio.ko || true



	if ((${#unsupported_nic_uio_hw[@]} > 0)); then

test/common/autobuild_common.sh

+1 −1
Original line number Diff line number Diff line
@@ -204,7 +204,7 @@ _build_native_dpdk() { 
		sudo -E ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS install

		# Sanitize ownership of the target directory post sudo above

		sudo chown -R "$USER" "$external_dpdk_base_dir"

		# Make sure kernel modules are available for freebsd_update_contigmem_mod() to fetch

		# Make sure kernel modules are available for freebsd_update_mods() to fetch

		mapfile -t drivers < <(find "$external_dpdk_base_dir/build-tmp" -name '*.ko')

		if ((${#drivers[@]} > 0)); then

			mkdir -p "$external_dpdk_dir/kmod"

test/common/autotest_common.sh

+28 −12
Original line number Diff line number Diff line
@@ -1451,20 +1451,36 @@ function autotest_cleanup() { 
}



function freebsd_update_contigmem_mod() {

	if [ $(uname) = FreeBSD ]; then

		kldunload contigmem.ko || true

		if [ -n "${SPDK_RUN_EXTERNAL_DPDK:-}" ]; then

			cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/modules/

			cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/contigmem.ko" /boot/kernel/

			cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/modules/

			cp -f "$SPDK_RUN_EXTERNAL_DPDK/kmod/nic_uio.ko" /boot/kernel/

		else

			cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/

			cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/

			cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/

			cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/

	local srcdir=${SPDK_RUN_EXTERNAL_DPDK:-"$rootdir/dpdk/build"}



	# Skip update if requested

	if [[ -n $USE_INSTALLED_CONTIGMEM ]]; then

		# If the driver is unloaded (for whatever reason) load it back in -

		# if it fails it means that driver is likely not around and/or the

		# sysctl setup is not available. In such a case, this is a hard fail.

		if ! kldstat -qm contigmem; then

			kldload contigmem || return 1

		fi

		echo "Skipping update of contigmem driver"

		return 0

	fi



	kldunload contigmem.ko || true

	cp -f "$srcdir/kmod/contigmem.ko" /boot/modules/

	cp -f "$srcdir/kmod/contigmem.ko" /boot/kernel/

}



function freebsd_update_nic_uio_mod() {

	local srcdir=${SPDK_RUN_EXTERNAL_DPDK:-"$rootdir/dpdk/build"}

	cp -f "$srcdir/kmod/nic_uio.ko" /boot/modules/

	cp -f "$srcdir/kmod/nic_uio.ko" /boot/kernel/

}



function freebsd_update_mods() {

	[[ $(uname) == FreeBSD ]] || return 0



	freebsd_update_contigmem_mod

	freebsd_update_nic_uio_mod

}



function freebsd_set_maxsock_buf() {

test/common/config/pkgdep/git

+47 −0
Original line number Diff line number Diff line
@@ -308,6 +308,50 @@ function install_igb_uio() { 
	sudo depmod

}



function install_contigmem() {

	# Dedicated solely for freebsd

	[[ $OSID == freebsd ]] || return 0



	rm -rf "$GIT_REPOS/dpdk-stable"



	git clone \

		--branch "$DPDK_STABLE_BRANCH" \

		--depth 1 "$DPDK_STABLE_REPO" \

		"$GIT_REPOS/dpdk-stable"



	# We skip most of the stuff to not waste time during setup but at the very end it won't

	# matter much as we want to build a single, specific target anyway. Note that DPDK considers

	# enable_kmods to be deprecated and as such, for freebsd kernel modules are always enabled,

	# so we don't need to set it in.

	meson setup \

		-Ddisable_apps="*" \

		-Ddisable_libs="*" \

		-Ddisable_drivers="*/*" \

		-Dtests=false \

		"$GIT_REPOS/dpdk-stable/build" \

		"$GIT_REPOS/dpdk-stable"



	# The thing we are after

	ninja -C "$GIT_REPOS/dpdk-stable/build" \

		kernel/freebsd/contigmem.ko



	[[ -e $GIT_REPOS/dpdk-stable/build/kernel/freebsd/contigmem.ko ]] || return 1



	# Put it into one of the default search paths

	cp -p \

		"$GIT_REPOS/dpdk-stable/build/kernel/freebsd/contigmem.ko" \

		"/boot/modules"



	# Make sure kernel loads it early on. Don't put it into the main loader file though, define

	# custom one. Also, provide a default setup of 2 * 1GB buffers - this should match default

	# hugepages requirement that autotest enforces under freebsd.

	cat <<-LOADER >/boot/loader.conf.d/spdk.conf

		contigmem_load="YES"

		hw.contigmem.num_buffers="2"

		hw.contigmem.buffer_size="$((1 << 30))"

	LOADER

}



function install_irdma() {

	local RDMA_CORE_VERSION=51.0

	local RDMA_CORE=https://github.com/linux-rdma/rdma-core/releases/download/v$RDMA_CORE_VERSION/rdma-core-$RDMA_CORE_VERSION.tar.gz
@@ -548,6 +592,7 @@ function install_sources() { 


	if [[ $OSID == freebsd ]]; then

		jobs=$(($(sysctl -n hw.ncpu) * 2))

		sources+=(install_contigmem)

	else

		jobs=$(($(nproc) * 2))

		sources+=(
@@ -588,6 +633,7 @@ ICE_VERSION=1.14.9 
BPFTRACE_VERSION=${BPFTRACE_VERSION:-f7bdfb44}

VFIO_QEMU_BRANCH=${VFIO_QEMU_BRANCH:-vfio-user-p3.0}

VANILLA_QEMU_BRANCH=${VANILLA_QEMU_BRANCH:-v8.0.0}

DPDK_STABLE_BRANCH=${DPDK_STABLE_BRANCH:-24.11}



: ${GIT_REPO_ROCKSDB=https://review.spdk.io/spdk/rocksdb}

export GIT_REPO_ROCKSDB
@@ -623,6 +669,7 @@ export GIT_REPO_ITTAPI 
export GIT_REPO_DOXYGEN

: ${GIT_REPO_LIBBPF="https://github.com/libbpf/libbpf"}

export GIT_REPO_LIBBPF

: ${DPDK_STABLE_REPO="http://dpdk.org/git/dpdk-stable"}



GIT_REPOS=${GIT_REPOS:-$HOME}

CC=${CC:-gcc}