Commit 0ce9b8aa authored by Seth Howell's avatar Seth Howell Committed by Daniel Verkamp
Browse files

rbd: add option for using installed version of ceph on rbd tests 



previously, the tests have been hardcoded to use a manually installed version of ceph.
The attached scripts allow the user to create a ceph cluster from a package managed version
of ceph which requires no extra configuration.

Change-Id: I22e4ea9eb95f1e84a968ee2bce6133568fda33a8
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/370399


Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 76414a00

scripts/autotest_common.sh

+8 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ fi 
: ${SPDK_TEST_ISCSI=1}; export SPDK_TEST_ISCSI

: ${SPDK_TEST_NVME=1}; export SPDK_TEST_NVME

: ${SPDK_TEST_NVMF=1}; export SPDK_TEST_NVMF

: ${SPDK_TEST_RBD=1}; export SPDK_TEST_RBD

: ${SPDK_TEST_VHOST=1}; export SPDK_TEST_VHOST

: ${SPDK_TEST_BLOCKDEV=1}; export SPDK_TEST_BLOCKDEV

: ${SPDK_TEST_IOAT=1}; export SPDK_TEST_IOAT
@@ -251,12 +252,19 @@ function rbd_setup() { 
		export RBD_NAME=foo

		(cd $CEPH_DIR && ../src/vstart.sh -d -n -x -l)

		/usr/local/bin/rbd create $RBD_NAME --size 1000

	elif hash ceph; then

		export RBD_POOL=rbd

		export RBD_NAME=foo

		(cd $rootdir/scripts/ceph && ./start.sh)

		rbd create $RBD_NAME --size 1000

	fi

}



function rbd_cleanup() {

	if [ -d $CEPH_DIR ]; then

		(cd $CEPH_DIR && ../src/stop.sh || true)

	elif hash ceph; then

		(cd $rootdir/scripts/ceph && ./stop.sh || true)

	fi

}

scripts/ceph/ceph.conf

0 → 100644
+67 −0
Original line number Diff line number Diff line 
[global]

        debug_lockdep = 0/0

        debug_context = 0/0

        debug_crush = 0/0

        debug_buffer = 0/0

        debug_timer = 0/0

        debug_filer = 0/0

        debug_objecter = 0/0

        debug_rados = 0/0

        debug_rbd = 0/0

        debug_ms = 0/0

        debug_monc = 0/0

        debug_tp = 0/0

        debug_auth = 0/0

        debug_finisher = 0/0

        debug_heartbeatmap = 0/0

        debug_perfcounter = 0/0

        debug_asok = 0/0

        debug_throttle = 0/0

        debug_mon = 0/0

        debug_paxos = 0/0

        debug_rgw = 0/0



        perf = true

        mutex_perf_counter = false

        throttler_perf_counter = false

        rbd cache = false

        mon_allow_pool_delete = true



        osd_pool_default_size = 1



[mon]

        mon_max_pool_pg_num=166496

        mon_osd_max_split_count = 10000

        mon_pg_warn_max_per_osd = 10000



[mon.a]

        mon addr = 127.0.0.1:12046



[osd]

        osd_op_threads = 64

        filestore_queue_max_ops=5000

        filestore_queue_committing_max_ops=5000

        journal_max_write_entries=1000

        journal_queue_max_ops=3000

        objecter_inflight_ops=102400

        filestore_wbthrottle_enable=false

        filestore_queue_max_bytes=1048576000

        filestore_queue_committing_max_bytes=1048576000

        journal_max_write_bytes=1048576000

        journal_queue_max_bytes=1048576000

        ms_dispatch_throttle_bytes=1048576000

        objecter_infilght_op_bytes=1048576000

        filestore_max_sync_interval=10

        osd_client_message_size_cap = 0

        osd_client_message_cap = 0

        osd_enable_op_tracker = false

        filestore_fd_cache_size = 10240

        filestore_fd_cache_shards = 64

        filestore_op_threads = 16

        osd_op_num_shards = 48

        osd_op_num_threads_per_shard = 2

        osd_pg_object_context_cache_count = 10240

        filestore_odsync_write = True

        journal_dynamic_throttle = True



[osd.0]

scripts/ceph/start.sh

0 → 100755
+95 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash

# create mon



set -x

set -e



fsid=`uuidgen`

base_dir=`pwd`

mon_ip=127.0.0.1

mon_dir=${base_dir}/ceph/mon.a/

pid_dir=${base_dir}/ceph/pid

ceph_conf=${base_dir}/ceph.conf

ceph_conf_temp=${base_dir}/ceph.conf.tmp

mnt_dir=${base_dir}/ceph/mnt

dev_backend=/dev/ceph

image=${base_dir}/../../../output/ceph_raw.img

dev=/dev/loop200

# partition osd



cp $ceph_conf $ceph_conf_temp



if [ ! -e $image ]; then

	fallocate -l 10G $image

fi



mknod ${dev_backend} b 7 200 || true

losetup ${dev_backend} ${image} || true



PARTED="parted -s"

SGDISK="sgdisk"



echo "Partitioning ${dev}"

${PARTED} ${dev} mktable gpt

sleep 2

${PARTED} ${dev} mkpart primary    0%    5GiB

${PARTED} ${dev} mkpart primary   5GiB  100%





partno=0

echo "Setting name on ${dev}"

${SGDISK} -c 1:osd-device-${partno}-journal ${dev}

${SGDISK} -c 2:osd-device-${partno}-data ${dev}

kpartx ${dev}



# prep osds



mnt_pt=${mnt_dir}/osd-device-0-data/

mkdir -p ${mnt_pt}

mkfs.xfs -f /dev/disk/by-partlabel/osd-device-0-data

mount /dev/disk/by-partlabel/osd-device-0-data ${mnt_pt}

echo -e "\tosd data = ${mnt_pt}" >> "$ceph_conf"

echo -e "\tosd journal = /dev/disk/by-partlabel/osd-device-0-journal" >> "$ceph_conf"





# create mon

rm -rf ${mon_dir}/*

mkdir -p ${mon_dir}

mkdir -p ${pid_dir}



ceph-authtool --create-keyring --gen-key --name=mon. ${base_dir}/ceph/keyring --cap mon 'allow *'

ceph-authtool --gen-key --name=client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' ${base_dir}/ceph/keyring



monmaptool --create --clobber --add a ${mon_ip}:12046 --print ${base_dir}/ceph/monmap



sh -c "ulimit -c unlimited && exec ceph-mon --mkfs -c ${ceph_conf} -i a --monmap=${base_dir}/ceph/monmap --keyring=${base_dir}/ceph/keyring --mon-data=${mon_dir}"



cp ${base_dir}/ceph/keyring ${mon_dir}/keyring



cp $ceph_conf /etc/ceph/ceph.conf



cp ${base_dir}/ceph/keyring /etc/ceph/keyring



ceph-run sh -c "ulimit -n 16384 && ulimit -c unlimited && exec ceph-mon -c ${ceph_conf} -i a --keyring=${base_dir}/ceph/keyring --pid-file=${base_dir}/ceph/pid/root@`hostname`.pid --mon-data=${mon_dir}" || true



# create osd



i=0



mkdir -p ${mnt_dir}



uuid=`uuidgen`

ceph -c ${ceph_conf} osd create ${uuid} $i

ceph-osd -c ${ceph_conf} -i $i --mkfs --mkkey --osd-uuid ${uuid}

ceph -c ${ceph_conf} osd crush add osd.${i} 1.0 host=`hostname` root=default

ceph -c ${ceph_conf} -i ${mnt_dir}/osd-device-${i}-data/keyring auth add osd.${i} osd "allow *" mon "allow profile osd" mgr "allow"



# start osd

pkill -9 ceph-osd || true

sleep 2



mkdir -p ${pid_dir}

env -i TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728 ceph-osd -c ${ceph_conf} -i 0 --pid-file=${pid_dir}/ceph-osd.0.pid



rm -f $ceph_conf

mv $ceph_conf_temp $ceph_conf

scripts/ceph/stop.sh

0 → 100755
+12 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash



set -x



base_dir=`pwd`

home_folder=${base_dir}/ceph

image=${base_dir}/../../../output/ceph_raw.img

pkill -9 ceph

sleep 3

umount /dev/loop200p2

rm -rf $home_folder

rm $image

test/iscsi_tgt/iscsi_tgt.sh

+3 −1
Original line number Diff line number Diff line
@@ -36,7 +36,9 @@ if [ $RUN_NIGHTLY -eq 1 ]; then 
	run_test ./test/iscsi_tgt/ip_migration/ip_migration.sh

fi

run_test ./test/iscsi_tgt/ext4test/ext4test.sh

if [ $SPDK_TEST_RBD -eq 1 ]; then

	run_test ./test/iscsi_tgt/rbd/rbd.sh

fi



trap - SIGINT SIGTERM EXIT

kill_stub