Compress test execution logs

The volume of logs we collect in CI jobs are extensive. This
patch ensures that the logs collected are compressed to reduce
the space taken in CI systems.

PYTHONUNBUFFERED is also set to ensure that the console log
from the CI jobs is recorded in the exact order of execution.

Change-Id: Ie13ecef2958a926933520cceeee6bfb669b94fce
Related-Bug: #1620849
This commit is contained in:
Jesse Pretorius 2016-09-09 10:48:44 +01:00
parent 80d2d96a5f
commit 1690a5d512
2 changed files with 26 additions and 7 deletions

View File

@ -36,3 +36,6 @@ python-ndg-httpsclient [platform:ubuntu !platform:ubuntu-trusty]
python2-pyasn1 [platform:rpm] python2-pyasn1 [platform:rpm]
pyOpenSSL [platform:rpm] pyOpenSSL [platform:rpm]
python-ndg_httpsclient [platform:rpm] python-ndg_httpsclient [platform:rpm]
# Required for compressing collected log files in CI
gzip

30
tox.ini
View File

@ -27,6 +27,7 @@ whitelist_externals =
wget wget
setenv = setenv =
VIRTUAL_ENV={envdir} VIRTUAL_ENV={envdir}
PYTHONUNBUFFERED=1
[testenv:docs] [testenv:docs]
@ -155,9 +156,17 @@ install_command =
pip install -U --force-reinstall {opts} {packages} pip install -U --force-reinstall {opts} {packages}
[testenv:func_logs]
commands =
bash -c 'mkdir -p {toxinidir}/logs'
bash -c 'rsync --archive --verbose --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
bash -c 'find "{toxinidir}/logs/" -type f | sed "p;s|$|.txt|" | xargs -n2 mv'
bash -c 'command gzip --best --recursive "{toxinidir}/logs/"'
[testenv:functional] [testenv:functional]
# Ignore_errors is set to true so that the logs are collected at the # Ignore_errors is set to true so that the logs are collected at the
# end of the run. This will not produce a failse positive. Any # end of the run. This will not produce a false positive. Any
# exception will be mark the run as failed and exit 1 after all of # exception will be mark the run as failed and exit 1 after all of
# the commands have been iterated through. # the commands have been iterated through.
ignore_errors = True ignore_errors = True
@ -180,12 +189,16 @@ commands =
-e "install_test_packages=True" \ -e "install_test_packages=True" \
-e @{toxinidir}/tests/test-vars.yml \ -e @{toxinidir}/tests/test-vars.yml \
{toxinidir}/tests/test-swift-functional.yml -vvvv {toxinidir}/tests/test-swift-functional.yml -vvvv
bash -c 'mkdir -p {toxinidir}/logs' {[testenv:func_logs]commands}
bash -c 'rsync -av --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
# NOTE(andymccr): this will test Swift running in pypy. # NOTE(andymccr): this will test Swift running in pypy.
[testenv:func_pypy] [testenv:func_pypy]
# Ignore_errors is set to true so that the logs are collected at the
# end of the run. This will not produce a false positive. Any
# exception will be mark the run as failed and exit 1 after all of
# the commands have been iterated through.
ignore_errors = True
install_command = install_command =
{[testenv:func_base]install_command} {[testenv:func_base]install_command}
deps = deps =
@ -206,12 +219,16 @@ commands =
-e @{toxinidir}/tests/test-vars.yml \ -e @{toxinidir}/tests/test-vars.yml \
-e "swift_pypy_enabled=True" \ -e "swift_pypy_enabled=True" \
{toxinidir}/tests/test-swift-functional.yml -vvvv {toxinidir}/tests/test-swift-functional.yml -vvvv
bash -c 'mkdir -p {toxinidir}/logs' {[testenv:func_logs]commands}
bash -c 'rsync -av --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
# NOTE(andmccr): this will test a MR Swift deploy. # NOTE(andmccr): this will test a MR Swift deploy.
[testenv:multi_reg] [testenv:multi_reg]
# Ignore_errors is set to true so that the logs are collected at the
# end of the run. This will not produce a false positive. Any
# exception will be mark the run as failed and exit 1 after all of
# the commands have been iterated through.
ignore_errors = True
install_command = install_command =
{[testenv:func_base]install_command} {[testenv:func_base]install_command}
deps = deps =
@ -266,8 +283,7 @@ commands =
-e "install_test_packages=True" \ -e "install_test_packages=True" \
-e @{toxinidir}/tests/test-vars-MR-B.yml \ -e @{toxinidir}/tests/test-vars-MR-B.yml \
{toxinidir}/tests/test-swift-functional.yml -vvvv {toxinidir}/tests/test-swift-functional.yml -vvvv
bash -c 'mkdir -p {toxinidir}/logs' {[testenv:func_logs]commands}
bash -c 'rsync -av --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
[testenv:linters] [testenv:linters]