diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 0fbe07b1..dd1a7a27 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -241,10 +241,17 @@ else $WORKSPACE/logs/devstack-gate-setup-workspace-new.txt fi +# relocate and symlink logs into $BASE to save space on the root filesystem +if [ -d "$WORKSPACE/logs" -a \! -e "$BASE/logs" ]; then + sudo mv $WORKSPACE/logs $BASE/ + ln -s $BASE/logs $WORKSPACE/ +fi + # Run pre test hook if we have one if function_exists "pre_test_hook"; then set -o xtrace - pre_test_hook 2>&1 | tee $WORKSPACE/logs/devstack-gate-pre-test-hook.txt + pre_test_hook 2>&1 | tee $WORKSPACE/devstack-gate-pre-test-hook.txt + sudo mv $WORKSPACE/devstack-gate-pre-test-hook.txt $BASE/logs/ set +o xtrace fi @@ -256,8 +263,9 @@ RETVAL=$GATE_RETVAL # Run post test hook if we have one if [ $GATE_RETVAL -eq 0 ] && function_exists "post_test_hook"; then set -o xtrace -o pipefail - post_test_hook 2>&1 | tee $WORKSPACE/logs/devstack-gate-post-test-hook.txt + post_test_hook 2>&1 | tee $WORKSPACE/devstack-gate-post-test-hook.txt RETVAL=$? + sudo mv $WORKSPACE/devstack-gate-post-test-hook.txt $BASE/logs/ set +o xtrace +o pipefail fi @@ -267,6 +275,7 @@ if [ $GATE_RETVAL -eq 137 ] && [ -f $WORKSPACE/gate.pid ] ; then sudo kill -s 9 -${GATEPID} fi -cleanup_host &> $WORKSPACE/logs/devstack-gate-cleanup-host.txt +cleanup_host &> $WORKSPACE/devstack-gate-cleanup-host.txt +sudo mv $WORKSPACE/devstack-gate-cleanup-host.txt $BASE/logs/ exit $RETVAL diff --git a/functions.sh b/functions.sh index 2dea8a3b..6d181549 100644 --- a/functions.sh +++ b/functions.sh @@ -341,88 +341,86 @@ function cleanup_host { # Sleep to give services a chance to flush their log buffers. sleep 2 - sudo cp /var/log/syslog $WORKSPACE/logs/syslog.txt - sudo cp /var/log/kern.log $WORKSPACE/logs/kern_log.txt - sudo cp /var/log/apache2/horizon_error.log $WORKSPACE/logs/horizon_error.log - mkdir $WORKSPACE/logs/rabbitmq/ - sudo cp /var/log/rabbitmq/* $WORKSPACE/logs/rabbitmq/ - mkdir $WORKSPACE/logs/sudoers.d/ + sudo cp /var/log/syslog $BASE/logs/syslog.txt + sudo cp /var/log/kern.log $BASE/logs/kern_log.txt + sudo cp /var/log/apache2/horizon_error.log $BASE/logs/horizon_error.log + sudo mkdir $BASE/logs/rabbitmq/ + sudo cp /var/log/rabbitmq/* $BASE/logs/rabbitmq/ + sudo mkdir $BASE/logs/sudoers.d/ - sudo cp /etc/sudoers.d/* $WORKSPACE/logs/sudoers.d/ - sudo cp /etc/sudoers $WORKSPACE/logs/sudoers.txt + sudo cp /etc/sudoers.d/* $BASE/logs/sudoers.d/ + sudo cp /etc/sudoers $BASE/logs/sudoers.txt if [ -d $BASE/old ]; then - mkdir -p $WORKSPACE/logs/old/ - mkdir -p $WORKSPACE/logs/new/ - mkdir -p $WORKSPACE/logs/grenade/ - sudo cp $BASE/old/screen-logs/* $WORKSPACE/logs/old/ - sudo cp $BASE/old/devstacklog.txt $WORKSPACE/logs/old/ - sudo cp $BASE/old/devstack/localrc $WORKSPACE/logs/old/localrc.txt - sudo cp $BASE/logs/* $WORKSPACE/logs/ - sudo cp $BASE/new/grenade/localrc $WORKSPACE/logs/grenade/localrc.txt - NEWLOGTARGET=$WORKSPACE/logs/new + sudo mkdir -p $BASE/logs/old $BASE/logs/new $BASE/logs/grenade + sudo cp $BASE/old/screen-logs/* $BASE/logs/old/ + sudo cp $BASE/old/devstacklog.txt $BASE/logs/old/ + sudo cp $BASE/old/devstack/localrc $BASE/logs/old/localrc.txt + sudo cp $BASE/new/grenade/localrc $BASE/logs/grenade/localrc.txt + NEWLOGTARGET=$BASE/logs/new else - NEWLOGTARGET=$WORKSPACE/logs + NEWLOGTARGET=$BASE/logs fi sudo cp $BASE/new/screen-logs/* $NEWLOGTARGET/ sudo cp $BASE/new/devstacklog.txt $NEWLOGTARGET/ sudo cp $BASE/new/devstack/localrc $NEWLOGTARGET/localrc.txt - sudo iptables-save > $WORKSPACE/logs/iptables.txt - df -h> $WORKSPACE/logs/df.txt - - pip freeze > $WORKSPACE/logs/pip-freeze.txt + sudo iptables-save > $WORKSPACE/iptables.txt + df -h> $WORKSPACE/df.txt + pip freeze > $WORKSPACE/pip-freeze.txt + sudo mv $WORKSPACE/iptables.txt $WORKSPACE/df.txt \ + $WORKSPACE/pip-freeze.txt $BASE/logs/ # Process testr artifacts. if [ -f $BASE/new/tempest/.testrepository/0 ]; then - sudo cp $BASE/new/tempest/.testrepository/0 $WORKSPACE/subunit_log.txt - sudo python /usr/local/jenkins/slave_scripts/subunit2html.py $WORKSPACE/subunit_log.txt $WORKSPACE/testr_results.html - sudo gzip -9 $WORKSPACE/subunit_log.txt - sudo gzip -9 $WORKSPACE/testr_results.html - sudo chown jenkins:jenkins $WORKSPACE/subunit_log.txt.gz $WORKSPACE/testr_results.html.gz - sudo chmod a+r $WORKSPACE/subunit_log.txt.gz $WORKSPACE/testr_results.html.gz + sudo cp $BASE/new/tempest/.testrepository/0 $BASE/logs/subunit_log.txt + sudo python /usr/local/jenkins/slave_scripts/subunit2html.py $BASE/logs/subunit_log.txt $BASE/logs/testr_results.html + sudo gzip -9 $BASE/logs/subunit_log.txt + sudo gzip -9 $BASE/logs/testr_results.html + sudo chown jenkins:jenkins $BASE/logs/subunit_log.txt.gz $BASE/logs/testr_results.html.gz + sudo chmod a+r $BASE/logs/subunit_log.txt.gz $BASE/logs/testr_results.html.gz elif [ -f $BASE/new/tempest/.testrepository/tmp* ]; then # If testr timed out, collect temp file from testr - sudo cp $BASE/new/tempest/.testrepository/tmp* $WORKSPACE/subunit_log.txt - sudo gzip -9 $WORKSPACE/subunit_log.txt - sudo chown jenkins:jenkins $WORKSPACE/subunit_log.txt.gz - sudo chmod a+r $WORKSPACE/subunit_log.txt.gz + sudo cp $BASE/new/tempest/.testrepository/tmp* $BASE/logs/subunit_log.txt + sudo gzip -9 $BASE/logs/subunit_log.txt + sudo chown jenkins:jenkins $BASE/logs/subunit_log.txt.gz + sudo chmod a+r $BASE/logs/subunit_log.txt.gz fi if [ -f $BASE/new/tempest/tempest.log ] ; then - sudo cp $BASE/new/tempest/tempest.log $WORKSPACE/logs/tempest.log + sudo cp $BASE/new/tempest/tempest.log $BASE/logs/tempest.log fi # Make sure jenkins can read all the logs - sudo chown -R jenkins:jenkins $WORKSPACE/logs/ - sudo chmod a+r $WORKSPACE/logs/ + sudo chown -R jenkins:jenkins $BASE/logs/ + sudo chmod a+r $BASE/logs/ - rename 's/\.log$/.txt/' $WORKSPACE/logs/* - rename 's/(.*)/$1.txt/' $WORKSPACE/logs/sudoers.d/* - rename 's/\.log$/.txt/' $WORKSPACE/logs/rabbitmq/* + rename 's/\.log$/.txt/' $BASE/logs/* + rename 's/(.*)/$1.txt/' $BASE/logs/sudoers.d/* + rename 's/\.log$/.txt/' $BASE/logs/rabbitmq/* - mv $WORKSPACE/logs/rabbitmq/startup_log \ - $WORKSPACE/logs/rabbitmq/startup_log.txt + sudo mv $BASE/logs/rabbitmq/startup_log \ + $BASE/logs/rabbitmq/startup_log.txt # Remove duplicate logs - rm $WORKSPACE/logs/*.*.txt + sudo rm $BASE/logs/*.*.txt if [ -d $BASE/old ]; then - rename 's/\.log$/.txt/' $WORKSPACE/logs/old/* - rename 's/\.log$/.txt/' $WORKSPACE/logs/new/* - rename 's/\.log$/.txt/' $WORKSPACE/logs/grenade/* - rm $WORKSPACE/logs/old/*.*.txt - rm $WORKSPACE/logs/new/*.*.txt + sudo rename 's/\.log$/.txt/' $BASE/logs/old/* + sudo rename 's/\.log$/.txt/' $BASE/logs/new/* + sudo rename 's/\.log$/.txt/' $BASE/logs/grenade/* + sudo rm $BASE/logs/old/*.*.txt + sudo rm $BASE/logs/new/*.*.txt fi # Compress all text logs - find $WORKSPACE/logs -iname '*.txt' -execdir gzip -9 {} \+ - find $WORKSPACE/logs -iname '*.dat' -execdir gzip -9 {} \+ + sudo find $BASE/logs -iname '*.txt' -execdir gzip -9 {} \+ + sudo find $BASE/logs -iname '*.dat' -execdir gzip -9 {} \+ # Save the tempest nosetests results - sudo cp $BASE/new/tempest/nosetests*.xml $WORKSPACE/ - sudo chown jenkins:jenkins $WORKSPACE/nosetests*.xml - sudo chmod a+r $WORKSPACE/nosetests*.xml + sudo cp $BASE/new/tempest/nosetests*.xml $BASE/logs/ + sudo chown jenkins:jenkins $BASE/logs/nosetests*.xml + sudo chmod a+r $BASE/logs/nosetests*.xml # Disable detailed logging as we return to the main script set +o xtrace