From 2a620be9b07799d66074440d2323c9c0c92091b5 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Tue, 21 May 2013 14:58:06 -0400 Subject: [PATCH] Add TOCI_GIT_CHECKOUT option. Added a new option to control whether toci checks out the various tripleO git repos it uses (incubator, diskimage_builder, etc.). Also, internally reworks the git clone directory names we so they just use the project names and not the GitHub org names as well. These changes should facilitate easier integration within the upstream jenkins. --- README.md | 1 + toci.sh | 27 +++++++++++++++++++-------- toci_functions.sh | 4 ++-- toci_git.sh | 25 +++++++++++++++++++++++++ toci_setup.sh | 29 ++++++++--------------------- toci_test.sh | 8 ++++---- 6 files changed, 59 insertions(+), 35 deletions(-) create mode 100755 toci_git.sh diff --git a/README.md b/README.md index ea4bf7855..fbe0644a7 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ TOCI_UPLOAD=0 TOCI_RESULTS_SERVER=1.2.3.4 TOCI_CLEANUP=1 TOCI_REMOVE=1 +TOCI_GIT_CHECKOUT=1 export http_proxy=http://1.2.3.4:3128 export https_proxy=http://1.2.3.4:3128 ``` diff --git a/toci.sh b/toci.sh index 8c55c98bc..c2a0ed412 100755 --- a/toci.sh +++ b/toci.sh @@ -5,10 +5,18 @@ export STARTTIME=$(date) export TOCI_SOURCE_DIR=$PWD +# env specific to this run, can contain +# TOCI_RESULTS_SERVER, http_proxy, TOCI_UPLOAD, TOCI_REMOVE, +source ~/.toci + +export TOCI_GIT_CHECKOUT + # All temp files should go here -export TOCI_WORKING_DIR=$(mktemp -d --tmpdir toci_working_XXXXXXX) +export TOCI_WORKING_DIR=${TOCI_WORKING_DIR:-$(mktemp -d --tmpdir toci_working_XXXXXXX)} +mkdir -p $TOCI_WORKING_DIR # Any files to be uploaded to results server goes here export TOCI_LOG_DIR=${TOCI_LOG_DIR:-$(mktemp -d --tmpdir toci_logs_XXXXXXX)} +mkdir -p $TOCI_LOG_DIR # Files that should be cached between runs should go in here # e.g. downloaded images, git repo's etc... export TOCI_CACHE_DIR=/var/tmp/toci_cache @@ -17,26 +25,29 @@ RESULT_CACHE=$TOCI_CACHE_DIR/results_cache.html echo "Starting run $STARTTIME ($TOCI_WORKING_DIR,$TOCI_LOG_DIR)" -# env specific to this run, can contain -# TOCI_RESULTS_SERVER, http_proxy, TOCI_UPLOAD, TOCI_REMOVE, -source ~/.toci # If running in cron $USER isn't setup export USER=${USER:-$(whoami)} mkdir -p $TOCI_CACHE_DIR STATUS=0 -mark_time Starting setup -timeout --foreground 60m ./toci_setup.sh > $TOCI_LOG_DIR/setup.out 2>&1 || STATUS=1 + +mark_time Starting git +timeout --foreground 60m ./toci_git.sh > $TOCI_LOG_DIR/git.out 2>&1 || STATUS=1 + if [ $STATUS == 0 ] ; then - mark_time Starting tests + mark_time Starting setup + timeout --foreground 60m ./toci_setup.sh > $TOCI_LOG_DIR/setup.out 2>&1 || STATUS=1 +fi +if [ $STATUS == 0 ] ; then + mark_time Starting test timeout --foreground 60m ./toci_test.sh > $TOCI_LOG_DIR/test.out 2>&1 || STATUS=1 fi if [ ${TOCI_CLEANUP:-1} == 1 ] ; then mark_time Starting cleanup timeout --foreground 60m ./toci_cleanup.sh > $TOCI_LOG_DIR/cleanup.out 2>&1 || STATUS=1 fi -mark_time Starting finished +mark_time Finished if [ ${TOCI_UPLOAD:-0} == 1 ] ; then cd $(dirname $TOCI_LOG_DIR) diff --git a/toci_functions.sh b/toci_functions.sh index a8ba47342..958b6c31b 100644 --- a/toci_functions.sh +++ b/toci_functions.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash get_get_repo(){ - CACHDIR=$TOCI_CACHE_DIR/${1/\//_} + CACHDIR=$TOCI_CACHE_DIR/${1/[^\/]*\//} if [ ! -e $CACHDIR ] ; then git clone https://github.com/$1.git $CACHDIR else @@ -9,7 +9,7 @@ get_get_repo(){ git fetch git reset --hard origin/master fi - cp -r $CACHDIR $TOCI_WORKING_DIR/${1/\//_} + cp -r $CACHDIR $TOCI_WORKING_DIR/${1/[^\/]*\//} } ssh_noprompt(){ diff --git a/toci_git.sh b/toci_git.sh new file mode 100755 index 000000000..1e2c794be --- /dev/null +++ b/toci_git.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -xe +. toci_functions.sh + +# Get the tripleO repo's +for repo in 'tripleo/incubator' 'tripleo/bm_poseur' 'stackforge/diskimage-builder' 'stackforge/tripleo-image-elements' ; do + if [ ${TOCI_GIT_CHECKOUT:-1} == 1 ] ; then + get_get_repo $repo + else + if [ ! -d "$TOCI_WORK_DIR/$repo" ]; then + echo "Please checkout $repo to $TOCI_WORK_DIR or enabled TOCI_GIT_CHECKOUT." + fi + fi +done + +#only patch if we do the git checkout +if [ ${TOCI_GIT_CHECKOUT:-1} == 1 ] ; then + # patches can be added to git repo's like this, this just a temp measure we need to make faster progress +# until we get up and runing properly + apply_patches incubator incubator* + apply_patches bm_poseur bm_poseur* + apply_patches diskimage-builder diskimage-builder* + apply_patches tripleo-image-elements tripleo-image-elements* +fi diff --git a/toci_setup.sh b/toci_setup.sh index fc919c8f3..016a7ae51 100755 --- a/toci_setup.sh +++ b/toci_setup.sh @@ -3,20 +3,8 @@ set -xe . toci_functions.sh -# Get the tripleO repo's -for repo in 'tripleo/incubator' 'tripleo/bm_poseur' 'stackforge/diskimage-builder' 'stackforge/tripleo-image-elements' ; do - get_get_repo $repo -done - -# patches can be added to git repo's like this, this just a temp measure we need to make faster progress -# until we get up and runing properly -apply_patches tripleo_incubator incubator* -apply_patches tripleo_bm_poseur bm_poseur* -apply_patches stackforge_diskimage-builder diskimage-builder* -apply_patches stackforge_tripleo-image-elements tripleo-image-elements* - # install deps on host machine -cd $TOCI_WORKING_DIR/tripleo_incubator +cd $TOCI_WORKING_DIR/incubator ./scripts/install-dependencies id | grep libvirt || ( echo "You have been added to the libvirt group, this script will now exit but will succeed if run again in a new shell" ; exit 1 ) @@ -24,7 +12,7 @@ id | grep libvirt || ( echo "You have been added to the libvirt group, this scri # looks like libvirt somtimes takes a little time to start wait_for 3 3 ls /var/run/libvirt/libvirt-sock -cd $TOCI_WORKING_DIR/tripleo_bm_poseur +cd $TOCI_WORKING_DIR/bm_poseur sudo ./bm_poseur --bridge-ip=none create-bridge || true if [ -f /etc/init.d/libvirt-bin ]; then @@ -33,14 +21,13 @@ else sudo service libvirtd restart fi -cd $TOCI_WORKING_DIR/stackforge_diskimage-builder/ -bin/disk-image-create -u base -a i386 -o $TOCI_WORKING_DIR/tripleo_incubator/base +cd $TOCI_WORKING_DIR/diskimage-builder/ +bin/disk-image-create -u base -a i386 -o $TOCI_WORKING_DIR/incubator/base - -cd $TOCI_WORKING_DIR/tripleo_incubator -sed -i "s/\"user\": \"stack\",/\"user\": \"`whoami`\",/" $TOCI_WORKING_DIR/stackforge_tripleo-image-elements/elements/boot-stack/config.json -ELEMENTS_PATH=$TOCI_WORKING_DIR/stackforge_tripleo-image-elements/elements \ -DIB_PATH=$TOCI_WORKING_DIR/stackforge_diskimage-builder \ +cd $TOCI_WORKING_DIR/incubator +sed -i "s/\"user\": \"stack\",/\"user\": \"`whoami`\",/" $TOCI_WORKING_DIR/tripleo-image-elements/elements/boot-stack/config.json +ELEMENTS_PATH=$TOCI_WORKING_DIR/tripleo-image-elements/elements \ +DIB_PATH=$TOCI_WORKING_DIR/diskimage-builder \ scripts/boot-elements boot-stack -o bootstrap BOOTSTRAP_IP=`scripts/get-vm-ip bootstrap` diff --git a/toci_test.sh b/toci_test.sh index 84d06db73..68962055f 100755 --- a/toci_test.sh +++ b/toci_test.sh @@ -3,7 +3,7 @@ set -xe . toci_functions.sh -cd $TOCI_WORKING_DIR/tripleo_incubator +cd $TOCI_WORKING_DIR/incubator BOOTSTRAP_IP=`scripts/get-vm-ip bootstrap` # Get logs from the node on exit @@ -16,14 +16,14 @@ source $TOCI_WORKING_DIR/stackrc unset http_proxy nova list -sudo $TOCI_WORKING_DIR/tripleo_bm_poseur/bm_poseur --vms 1 --arch i686 create-vm -MAC=`$TOCI_WORKING_DIR/tripleo_bm_poseur/bm_poseur get-macs` +sudo $TOCI_WORKING_DIR/bm_poseur/bm_poseur --vms 1 --arch i686 create-vm +MAC=`$TOCI_WORKING_DIR/bm_poseur/bm_poseur get-macs` nova keypair-add --pub-key ~/.ssh/id_rsa.pub default nova baremetal-node-create ubuntu 1 512 10 $MAC # Load the base image into glance -export DIB_PATH=$TOCI_WORKING_DIR/stackforge_diskimage-builder +export DIB_PATH=$TOCI_WORKING_DIR/diskimage-builder ./scripts/load-image base.qcow2 # place the bootstrap public key on host so that it can admin virt