diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index f5eb1fe4..685661f4 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # Gate commits to several projects on a VM running those projects # configured by devstack. @@ -38,112 +38,157 @@ if [ -z "$SKIP_DEVSTACK_GATE_PROJECT" ]; then PROJECTS="openstack-ci/devstack-gate $PROJECTS" fi -# HPcloud stopped adding the hostname to /etc/hosts with their -# precise images. +# Most of the work of this script is done in functions so that we may +# easily redirect their stdout / stderr to log files. -HOSTNAME=`/bin/hostname` -if ! grep $HOSTNAME /etc/hosts >/dev/null -then - echo "Need to add hostname to /etc/hosts" - sudo bash -c 'echo "127.0.1.1 $HOSTNAME" >>/etc/hosts' -fi +function setup_workspace { + # Enabled detailed logging, since output of this function is redirected + set -o xtrace -cd $WORKSPACE + # HPcloud stopped adding the hostname to /etc/hosts with their + # precise images. -ORIGINAL_GERRIT_PROJECT=$GERRIT_PROJECT -ORIGINAL_GERRIT_BRANCH=$GERRIT_BRANCH - -for GERRIT_PROJECT in $PROJECTS -do - echo "Setting up $GERRIT_PROJECT" - SHORT_PROJECT=`basename $GERRIT_PROJECT` - if [[ ! -e $SHORT_PROJECT ]]; then - echo " Need to clone" - git clone https://review.openstack.org/p/$GERRIT_PROJECT + HOSTNAME=`/bin/hostname` + if ! grep $HOSTNAME /etc/hosts >/dev/null + then + echo "Need to add hostname to /etc/hosts" + sudo bash -c 'echo "127.0.1.1 $HOSTNAME" >>/etc/hosts' fi - cd $SHORT_PROJECT - - GERRIT_BRANCH=$ORIGINAL_GERRIT_BRANCH - - # See if this project has this branch, if not, use master - git remote update || git remote update # attempt to work around bug #925790 - # Ensure that we don't have stale remotes around - git remote prune origin - if ! git branch -a |grep remotes/origin/$GERRIT_BRANCH>/dev/null; then - GERRIT_BRANCH=master - fi - - export GERRIT_BRANCH - export GERRIT_PROJECT - /usr/local/jenkins/slave_scripts/gerrit-git-prep.sh review.openstack.org cd $WORKSPACE -done -GERRIT_PROJECT=$ORIGINAL_GERRIT_PROJECT -GERRIT_BRANCH=$ORIGINAL_GERRIT_BRANCH + ORIGINAL_GERRIT_PROJECT=$GERRIT_PROJECT + ORIGINAL_GERRIT_BRANCH=$GERRIT_BRANCH -# Set GATE_SCRIPT_DIR to point to devstack-gate in the workspace so that -# we are testing the proposed change from this point forward. -GATE_SCRIPT_DIR=$WORKSPACE/devstack-gate + for GERRIT_PROJECT in $PROJECTS + do + echo "Setting up $GERRIT_PROJECT" + SHORT_PROJECT=`basename $GERRIT_PROJECT` + if [[ ! -e $SHORT_PROJECT ]]; then + echo " Need to clone" + git clone https://review.openstack.org/p/$GERRIT_PROJECT + fi + cd $SHORT_PROJECT + + GERRIT_BRANCH=$ORIGINAL_GERRIT_BRANCH + + # See if this project has this branch, if not, use master + git remote update || git remote update # attempt to work around bug #925790 + # Ensure that we don't have stale remotes around + git remote prune origin + if ! git branch -a |grep remotes/origin/$GERRIT_BRANCH>/dev/null; then + GERRIT_BRANCH=master + fi + + export GERRIT_BRANCH + export GERRIT_PROJECT + /usr/local/jenkins/slave_scripts/gerrit-git-prep.sh review.openstack.org + + cd $WORKSPACE + done + + GERRIT_PROJECT=$ORIGINAL_GERRIT_PROJECT + GERRIT_BRANCH=$ORIGINAL_GERRIT_BRANCH + + # Set GATE_SCRIPT_DIR to point to devstack-gate in the workspace so that + # we are testing the proposed change from this point forward. + GATE_SCRIPT_DIR=$WORKSPACE/devstack-gate + + # Disable detailed logging as we return to the main script + set +o xtrace +} + +function setup_host { + # Enabled detailed logging, since output of this function is redirected + set -o xtrace + + # Make sure headers for the currently running kernel are installed: + sudo apt-get install -y --force-yes linux-headers-`uname -r` + + # Hpcloud provides no swap, but does have a partition mounted at /mnt + # we can use: + if [ `cat /proc/meminfo | grep SwapTotal | awk '{ print $2; }'` -eq 0 ] && + [ -b /dev/vdb ]; then + sudo umount /dev/vdb + sudo mkswap /dev/vdb + sudo swapon /dev/vdb + fi + + # The vm template update job should cache some images in ~/files. + # Move them to where devstack expects: + if ls ~/cache/files/*; then + mv ~/cache/files/* $WORKSPACE/devstack/files + fi + + # Move the PIP cache into position: + sudo mkdir -p /var/cache/pip + sudo mv ~/cache/pip/* /var/cache/pip + + # Start with a fresh syslog + sudo stop rsyslog + sudo mv /var/log/syslog /var/log/syslog-pre-devstack + sudo touch /var/log/syslog + sudo chown /var/log/syslog --ref /var/log/syslog-pre-devstack + sudo chmod /var/log/syslog --ref /var/log/syslog-pre-devstack + sudo chmod a+r /var/log/syslog + sudo start rsyslog + + # Disable detailed logging as we return to the main script + set +o xtrace +} + +function cleanup_host { + # Enabled detailed logging, since output of this function is redirected + set -o xtrace + + cd $WORKSPACE + # No matter what, archive logs + + sudo cp /var/log/syslog $WORKSPACE/logs/syslog.txt + cp $WORKSPACE/screen-logs/* $WORKSPACE/logs/ + + # Make the devstack localrc available with the logs + cp $WORKSPACE/devstack/localrc $WORKSPACE/logs/localrc.txt + + # Make sure jenkins can read all the logs + sudo chown -R jenkins.jenkins $WORKSPACE/logs/ + sudo chmod a+r $WORKSPACE/logs/ + + rename 's/\.log$/.txt/' $WORKSPACE/logs/* + + # Remove duplicate logs + rm $WORKSPACE/logs/*.*.txt + + # Disable detailed logging as we return to the main script + set +o xtrace +} + +# Make a directory to store logs +mkdir -p logs +rm -f logs/* + +setup_workspace &> $WORKSPACE/logs/devstack-gate-setup-workspace.txt # Also, if we're testing devstack-gate, re-exec this script once so # that we can test the new version of it. if [[ $GERRIT_PROJECT == "openstack-ci/devstack-gate" ]] && [[ $RE_EXEC != "true" ]]; then export RE_EXEC="true" + echo "This build includes a change to the devstack gate; re-execing this script." exec $GATE_SCRIPT_DIR/devstack-vm-gate-wrap.sh fi -# Make sure headers for the currently running kernel are installed: -sudo apt-get install -y --force-yes linux-headers-`uname -r` - -# Hpcloud provides no swap, but does have a partition mounted at /mnt -# we can use: -if [ `cat /proc/meminfo | grep SwapTotal | awk '{ print $2; }'` -eq 0 ] && - [ -b /dev/vdb ]; then - sudo umount /dev/vdb - sudo mkswap /dev/vdb - sudo swapon /dev/vdb +if [ ! -z "$GERRIT_CHANGES" ] +then + CHANGE_NUMBER=`echo $GERRIT_CHANGES|grep -Po ".*/\K\d+(?=/\d+)"` + echo "Triggered by: https://review.openstack.org/$CHANGE_NUMBER" fi -# The vm template update job should cache some images in ~/files. -# Move them to where devstack expects: -if ls ~/cache/files/*; then - mv ~/cache/files/* $WORKSPACE/devstack/files -fi - -# Move the PIP cache into position: -sudo mkdir -p /var/cache/pip -sudo mv ~/cache/pip/* /var/cache/pip - -# Start with a fresh syslog -sudo stop rsyslog -sudo mv /var/log/syslog /var/log/syslog-pre-devstack -sudo touch /var/log/syslog -sudo chown /var/log/syslog --ref /var/log/syslog-pre-devstack -sudo chmod /var/log/syslog --ref /var/log/syslog-pre-devstack -sudo chmod a+r /var/log/syslog -sudo start rsyslog +setup_host &> $WORKSPACE/logs/devstack-gate-setup-host.txt # Run the test $GATE_SCRIPT_DIR/devstack-vm-gate.sh RETVAL=$? -cd $WORKSPACE -# No matter what, archive logs -mkdir -p logs -rm -f logs/* - -sudo cp /var/log/syslog $WORKSPACE/logs/syslog.txt -cp $WORKSPACE/screen-logs/* $WORKSPACE/logs/ - -# Make sure jenkins can read all the logs -sudo chown -R jenkins.jenkins $WORKSPACE/logs/ -sudo chmod a+r $WORKSPACE/logs/ - -rename 's/\.log$/.txt/' $WORKSPACE/logs/* - -# Remove duplicate logs -rm $WORKSPACE/logs/*.*.txt +cleanup_host &> $WORKSPACE/logs/devstack-gate-cleanup-host.txt exit $RETVAL diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index cadfb807..41288d4c 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # Script that is run on the devstack vm; configures and # invokes devstack. @@ -21,9 +21,7 @@ set -o errexit -export DEST=$WORKSPACE - -cd $DEST/devstack +cd $WORKSPACE/devstack ENABLED_SERVICES=g-api,g-reg,key,n-api,n-crt,n-obj,n-cpu,n-net,n-vol,n-sch,horizon,mysql,rabbit @@ -61,11 +59,15 @@ if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then echo "VOLUME_BACKING_FILE_SIZE=4G" >> localrc fi +echo "Running devstack" ./stack.sh if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then - ./tools/configure_tempest.sh - cd $DEST/tempest - nosetests --with-xunit -sv $DEVSTACK_GATE_TEMPEST_TESTS + echo "Configuring tempest" + ./tools/configure_tempest.sh + cd $WORKSPACE/tempest + echo "Running tempest" + nosetests --with-xunit -sv $DEVSTACK_GATE_TEMPEST_TESTS else - ./exercise.sh + echo "Running devstack exercises" + ./exercise.sh fi