From db80d838770915b44df9c0a822ef9204b0b494f1 Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Thu, 23 Jan 2014 21:52:14 +0000 Subject: [PATCH] Keep logs in $BASE instead of $WORKSPACE Avoid moving logs into the jenkins homedir unnecessarily, since it has a tendency to fill up the root filesystem and the SCP publisher can follow symlinks out of the workspace anyway. We've seen slaves go offline due to full root filesystems recently, so this should hopefully reduce the incidence of that problem. Change-Id: I490ed53989137218f14464708a65969033c108d6 Closes-Bug: #1268732 --- devstack-vm-gate-wrap.sh | 15 ++++-- functions.sh | 100 +++++++++++++++++++-------------------- 2 files changed, 61 insertions(+), 54 deletions(-) diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 0fbe07b1..dd1a7a27 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -241,10 +241,17 @@ else $WORKSPACE/logs/devstack-gate-setup-workspace-new.txt fi +# relocate and symlink logs into $BASE to save space on the root filesystem +if [ -d "$WORKSPACE/logs" -a \! -e "$BASE/logs" ]; then + sudo mv $WORKSPACE/logs $BASE/ + ln -s $BASE/logs $WORKSPACE/ +fi + # Run pre test hook if we have one if function_exists "pre_test_hook"; then set -o xtrace - pre_test_hook 2>&1 | tee $WORKSPACE/logs/devstack-gate-pre-test-hook.txt + pre_test_hook 2>&1 | tee $WORKSPACE/devstack-gate-pre-test-hook.txt + sudo mv $WORKSPACE/devstack-gate-pre-test-hook.txt $BASE/logs/ set +o xtrace fi @@ -256,8 +263,9 @@ RETVAL=$GATE_RETVAL # Run post test hook if we have one if [ $GATE_RETVAL -eq 0 ] && function_exists "post_test_hook"; then set -o xtrace -o pipefail - post_test_hook 2>&1 | tee $WORKSPACE/logs/devstack-gate-post-test-hook.txt + post_test_hook 2>&1 | tee $WORKSPACE/devstack-gate-post-test-hook.txt RETVAL=$? + sudo mv $WORKSPACE/devstack-gate-post-test-hook.txt $BASE/logs/ set +o xtrace +o pipefail fi @@ -267,6 +275,7 @@ if [ $GATE_RETVAL -eq 137 ] && [ -f $WORKSPACE/gate.pid ] ; then sudo kill -s 9 -${GATEPID} fi -cleanup_host &> $WORKSPACE/logs/devstack-gate-cleanup-host.txt +cleanup_host &> $WORKSPACE/devstack-gate-cleanup-host.txt +sudo mv $WORKSPACE/devstack-gate-cleanup-host.txt $BASE/logs/ exit $RETVAL diff --git a/functions.sh b/functions.sh index 2dea8a3b..6d181549 100644 --- a/functions.sh +++ b/functions.sh @@ -341,88 +341,86 @@ function cleanup_host { # Sleep to give services a chance to flush their log buffers. sleep 2 - sudo cp /var/log/syslog $WORKSPACE/logs/syslog.txt - sudo cp /var/log/kern.log $WORKSPACE/logs/kern_log.txt - sudo cp /var/log/apache2/horizon_error.log $WORKSPACE/logs/horizon_error.log - mkdir $WORKSPACE/logs/rabbitmq/ - sudo cp /var/log/rabbitmq/* $WORKSPACE/logs/rabbitmq/ - mkdir $WORKSPACE/logs/sudoers.d/ + sudo cp /var/log/syslog $BASE/logs/syslog.txt + sudo cp /var/log/kern.log $BASE/logs/kern_log.txt + sudo cp /var/log/apache2/horizon_error.log $BASE/logs/horizon_error.log + sudo mkdir $BASE/logs/rabbitmq/ + sudo cp /var/log/rabbitmq/* $BASE/logs/rabbitmq/ + sudo mkdir $BASE/logs/sudoers.d/ - sudo cp /etc/sudoers.d/* $WORKSPACE/logs/sudoers.d/ - sudo cp /etc/sudoers $WORKSPACE/logs/sudoers.txt + sudo cp /etc/sudoers.d/* $BASE/logs/sudoers.d/ + sudo cp /etc/sudoers $BASE/logs/sudoers.txt if [ -d $BASE/old ]; then - mkdir -p $WORKSPACE/logs/old/ - mkdir -p $WORKSPACE/logs/new/ - mkdir -p $WORKSPACE/logs/grenade/ - sudo cp $BASE/old/screen-logs/* $WORKSPACE/logs/old/ - sudo cp $BASE/old/devstacklog.txt $WORKSPACE/logs/old/ - sudo cp $BASE/old/devstack/localrc $WORKSPACE/logs/old/localrc.txt - sudo cp $BASE/logs/* $WORKSPACE/logs/ - sudo cp $BASE/new/grenade/localrc $WORKSPACE/logs/grenade/localrc.txt - NEWLOGTARGET=$WORKSPACE/logs/new + sudo mkdir -p $BASE/logs/old $BASE/logs/new $BASE/logs/grenade + sudo cp $BASE/old/screen-logs/* $BASE/logs/old/ + sudo cp $BASE/old/devstacklog.txt $BASE/logs/old/ + sudo cp $BASE/old/devstack/localrc $BASE/logs/old/localrc.txt + sudo cp $BASE/new/grenade/localrc $BASE/logs/grenade/localrc.txt + NEWLOGTARGET=$BASE/logs/new else - NEWLOGTARGET=$WORKSPACE/logs + NEWLOGTARGET=$BASE/logs fi sudo cp $BASE/new/screen-logs/* $NEWLOGTARGET/ sudo cp $BASE/new/devstacklog.txt $NEWLOGTARGET/ sudo cp $BASE/new/devstack/localrc $NEWLOGTARGET/localrc.txt - sudo iptables-save > $WORKSPACE/logs/iptables.txt - df -h> $WORKSPACE/logs/df.txt - - pip freeze > $WORKSPACE/logs/pip-freeze.txt + sudo iptables-save > $WORKSPACE/iptables.txt + df -h> $WORKSPACE/df.txt + pip freeze > $WORKSPACE/pip-freeze.txt + sudo mv $WORKSPACE/iptables.txt $WORKSPACE/df.txt \ + $WORKSPACE/pip-freeze.txt $BASE/logs/ # Process testr artifacts. if [ -f $BASE/new/tempest/.testrepository/0 ]; then - sudo cp $BASE/new/tempest/.testrepository/0 $WORKSPACE/subunit_log.txt - sudo python /usr/local/jenkins/slave_scripts/subunit2html.py $WORKSPACE/subunit_log.txt $WORKSPACE/testr_results.html - sudo gzip -9 $WORKSPACE/subunit_log.txt - sudo gzip -9 $WORKSPACE/testr_results.html - sudo chown jenkins:jenkins $WORKSPACE/subunit_log.txt.gz $WORKSPACE/testr_results.html.gz - sudo chmod a+r $WORKSPACE/subunit_log.txt.gz $WORKSPACE/testr_results.html.gz + sudo cp $BASE/new/tempest/.testrepository/0 $BASE/logs/subunit_log.txt + sudo python /usr/local/jenkins/slave_scripts/subunit2html.py $BASE/logs/subunit_log.txt $BASE/logs/testr_results.html + sudo gzip -9 $BASE/logs/subunit_log.txt + sudo gzip -9 $BASE/logs/testr_results.html + sudo chown jenkins:jenkins $BASE/logs/subunit_log.txt.gz $BASE/logs/testr_results.html.gz + sudo chmod a+r $BASE/logs/subunit_log.txt.gz $BASE/logs/testr_results.html.gz elif [ -f $BASE/new/tempest/.testrepository/tmp* ]; then # If testr timed out, collect temp file from testr - sudo cp $BASE/new/tempest/.testrepository/tmp* $WORKSPACE/subunit_log.txt - sudo gzip -9 $WORKSPACE/subunit_log.txt - sudo chown jenkins:jenkins $WORKSPACE/subunit_log.txt.gz - sudo chmod a+r $WORKSPACE/subunit_log.txt.gz + sudo cp $BASE/new/tempest/.testrepository/tmp* $BASE/logs/subunit_log.txt + sudo gzip -9 $BASE/logs/subunit_log.txt + sudo chown jenkins:jenkins $BASE/logs/subunit_log.txt.gz + sudo chmod a+r $BASE/logs/subunit_log.txt.gz fi if [ -f $BASE/new/tempest/tempest.log ] ; then - sudo cp $BASE/new/tempest/tempest.log $WORKSPACE/logs/tempest.log + sudo cp $BASE/new/tempest/tempest.log $BASE/logs/tempest.log fi # Make sure jenkins can read all the logs - sudo chown -R jenkins:jenkins $WORKSPACE/logs/ - sudo chmod a+r $WORKSPACE/logs/ + sudo chown -R jenkins:jenkins $BASE/logs/ + sudo chmod a+r $BASE/logs/ - rename 's/\.log$/.txt/' $WORKSPACE/logs/* - rename 's/(.*)/$1.txt/' $WORKSPACE/logs/sudoers.d/* - rename 's/\.log$/.txt/' $WORKSPACE/logs/rabbitmq/* + rename 's/\.log$/.txt/' $BASE/logs/* + rename 's/(.*)/$1.txt/' $BASE/logs/sudoers.d/* + rename 's/\.log$/.txt/' $BASE/logs/rabbitmq/* - mv $WORKSPACE/logs/rabbitmq/startup_log \ - $WORKSPACE/logs/rabbitmq/startup_log.txt + sudo mv $BASE/logs/rabbitmq/startup_log \ + $BASE/logs/rabbitmq/startup_log.txt # Remove duplicate logs - rm $WORKSPACE/logs/*.*.txt + sudo rm $BASE/logs/*.*.txt if [ -d $BASE/old ]; then - rename 's/\.log$/.txt/' $WORKSPACE/logs/old/* - rename 's/\.log$/.txt/' $WORKSPACE/logs/new/* - rename 's/\.log$/.txt/' $WORKSPACE/logs/grenade/* - rm $WORKSPACE/logs/old/*.*.txt - rm $WORKSPACE/logs/new/*.*.txt + sudo rename 's/\.log$/.txt/' $BASE/logs/old/* + sudo rename 's/\.log$/.txt/' $BASE/logs/new/* + sudo rename 's/\.log$/.txt/' $BASE/logs/grenade/* + sudo rm $BASE/logs/old/*.*.txt + sudo rm $BASE/logs/new/*.*.txt fi # Compress all text logs - find $WORKSPACE/logs -iname '*.txt' -execdir gzip -9 {} \+ - find $WORKSPACE/logs -iname '*.dat' -execdir gzip -9 {} \+ + sudo find $BASE/logs -iname '*.txt' -execdir gzip -9 {} \+ + sudo find $BASE/logs -iname '*.dat' -execdir gzip -9 {} \+ # Save the tempest nosetests results - sudo cp $BASE/new/tempest/nosetests*.xml $WORKSPACE/ - sudo chown jenkins:jenkins $WORKSPACE/nosetests*.xml - sudo chmod a+r $WORKSPACE/nosetests*.xml + sudo cp $BASE/new/tempest/nosetests*.xml $BASE/logs/ + sudo chown jenkins:jenkins $BASE/logs/nosetests*.xml + sudo chmod a+r $BASE/logs/nosetests*.xml # Disable detailed logging as we return to the main script set +o xtrace