openstack/kolla
Code Issues Proposed changes

Clean up test process

Browse Source 
If stestr failed, then no node info was logged.
Logging node info is now handled in pre and post stages of Zuul.
Additionally Zuul v3 migration recommendations were applied
and some unused code removed improving general readability.

Change-Id: Iae9d37cb76997ba8389e21c7e2a3bf5905abe029
Signed-off-by: Radosław Piliszek <radoslaw.piliszek@gmail.com>
This commit is contained in:
Radosław Piliszek 2019-06-11 15:10:13 +02:00
parent 8690768880
commit c7ac0f0de0
10 changed files with 62 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.zuul.d/base.yaml
View File

@ -46,9 +46,6 @@
run: tests/playbooks/run.yml
post-run: tests/playbooks/post.yml
attempts: 1
required-projects:
- openstack/kolla-ansible
- openstack/requirements
irrelevant-files:
- ^.*\.rst$
- ^doc/.*
@ -59,3 +56,6 @@
- ^test-requirements.txt$
vars:
publisher: false
extra-vars:
kolla_logs_dir: "{{ zuul_output_dir }}/logs/kolla"
kolla_build_logs_dir: "{{ kolla_logs_dir }}/build"

83
tests/playbooks/post.yml
View File

@ -1,60 +1,66 @@
---
- hosts: all
tasks:
- shell:
- name: Dump host info to logs
command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
args:
chdir: "{{ kolla_logs_dir }}"
- name: Collect various info to logs
shell:
cmd: |
set +o errexit
mkdir -p logs
# copy system log
set -o xtrace
sudo cp -r /var/log logs/system_log
# copy system log
sudo cp -r /var/log system_log
if which journalctl ; then
# the journal gives us syslog() and kernel output, so is like
# a concatenation of the above.
sudo journalctl --no-pager | sudo tee logs/syslog.txt > /dev/null
sudo journalctl --no-pager -u docker.service | sudo tee logs/docker.log > /dev/null
sudo journalctl --no-pager > syslog.txt
sudo journalctl --no-pager -u docker.service > docker.log
else
# assume rsyslog
sudo cp /var/log/syslog logs/syslog.txt
sudo cp /var/log/kern.log logs/kern_log.txt
sudo cp /var/log/upstart/docker.log logs/docker.log
sudo cp /var/log/syslog syslog.txt
sudo cp /var/log/kern.log kern_log.txt
sudo cp /var/log/upstart/docker.log docker.log
fi
if sudo test -d /var/lib/docker/volumes/kolla_logs/_data; then
sudo cp -r /var/lib/docker/volumes/kolla_logs/_data logs/kolla_logs
sudo cp -r /var/lib/docker/volumes/kolla_logs/_data kolla_logs
fi
# sudo config
sudo cp -r /etc/sudoers.d logs/
sudo cp /etc/sudoers logs/sudoers.txt
sudo cp -r /etc/sudoers.d ./
sudo cp /etc/sudoers sudoers.txt
df -h > logs/df.txt
free > logs/free.txt
sudo parted -l | sudo tee logs/parted-l.txt > /dev/null
mount > logs/mount.txt
env > logs/env.txt
df -h > df.txt
free > free.txt
sudo parted -l > parted-l.txt
mount > mount.txt
env > env.txt
if [ `command -v dpkg` ]; then
dpkg -l | sudo tee logs/dpkg-l.txt > /dev/null
dpkg -l > dpkg-l.txt
fi
if [ `command -v rpm` ]; then
rpm -qa | sudo tee logs/rpm-qa.txt > /dev/null
rpm -qa > rpm-qa.txt
fi
# final memory usage and process list
ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > logs/ps.txt
ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ps.txt
# docker related information
(docker info && docker system df && docker system df -v) > logs/docker-info.txt
(docker info && docker system df && docker system df -v) > docker-info.txt
sudo cp -r /etc/kolla logs/kolla_configs
sudo cp -r /etc/kolla kolla_configs
# fix the permissions for logs folder
sudo chmod -R 777 logs
# last sudo to own all files
sudo chown -R {{ ansible_user_id }}: .
# Parse build logs and extract pkg install info
tests/files/process_build_logs.py -l logs/build -b {{ base_distro }} > logs/packages-info.txt
{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py -l build -b {{ base_distro }} > packages-info.txt
# rename files to .txt; this is so that when displayed via
# logs.openstack.org clicking results in the browser shows the
@ -62,30 +68,21 @@
# download it, etc.
# firstly, rename all .log files to .txt files
for f in $(find logs -name "*.log"); do
sudo mv $f ${f/.log/.txt}
for f in $(find . -name "*.log"); do
mv $f ${f/.log/.txt}
done
# Update symlinks to new file names
for f in $(find logs -name "*FAILED*"); do
sudo mv ${f} ${f}.gz
sudo ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
for f in $(find . -name "*FAILED*"); do
mv ${f} ${f}.gz
ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
done
# append .txt to all kolla config file
find logs/kolla_configs -type f -exec mv '{}' '{}'.txt \;
find kolla_configs -type f -exec mv '{}' '{}'.txt \;
# Compress all text logs
find logs -iname '*.txt' -execdir gzip -9 {} \+
find logs -iname '*.json' -execdir gzip -9 {} \+
find . -iname '*.txt' -execdir gzip -9 {} \+
find . -iname '*.json' -execdir gzip -9 {} \+
executable: /bin/bash
chdir: "{{ zuul.project.src_dir }}"
- synchronize:
src: '{{ zuul.project.src_dir }}/logs'
dest: '{{ zuul.executor.log_root }}'
mode: pull
copy_links: true
verify_host: true
rsync_opts:
- "--quiet"
chdir: "{{ kolla_logs_dir }}"

22
tests/playbooks/pre.yml
View File

@ -1,13 +1,17 @@
---
- hosts: all
become: true
tasks:
- name: Install dbus for debian system
apt: name=dbus
when: ansible_os_family == 'Debian'
- name: Create dir for kolla logs
file:
path: "{{ kolla_logs_dir }}"
state: directory
- name: Install ansible
pip:
name: ansible
when:
- inventory_hostname == "primary"
- name: Dump host info to logs
command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
args:
chdir: "{{ kolla_logs_dir }}"
- name: Create dir for kolla build logs
file:
path: "{{ kolla_build_logs_dir }}"
state: directory

35
tests/playbooks/run.yml
View File

@ -1,7 +1,6 @@
---
- hosts: all
vars:
kolla_inventory_path: "{{ zuul.project.src_dir }}/inventory"
nodepool_rdo_proxy: "{{ zuul_site_mirror_fqdn }}:8080/rdo"
nodepool_pypi_mirror: "http://{{ zuul_site_mirror_fqdn }}:8080/pypi/simple"
nodepool_mirror_host: "{{ zuul_site_mirror_fqdn }}"
@ -12,23 +11,6 @@
nodepool_oraclelinux_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/oraclelinux/"
nodepool_percona_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/percona"
tasks:
- name: generate kolla inventory file
copy:
dest: "{{ kolla_inventory_path }}"
content: |
{% for host in hostvars %}
{{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user={{ hostvars[host]['ansible_user'] }}
{% endfor %}
- name: setup logging
shell:
cmd: |
mkdir logs
ln -s $(pwd)/logs /tmp/logs
mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs}
executable: /bin/bash
chdir: "{{ zuul.project.src_dir }}"
- name: copy setup script
copy:
src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tools/setup_{{ ansible_os_family }}.sh"
@ -57,21 +39,12 @@
src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2"
dest: /etc/kolla/kolla-build.conf
- name: Template override
- name: Template template_overrides.j2
template:
src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/template_overrides.j2"
dest: /etc/kolla/template_overrides.j2
- shell:
cmd: |
set -e
set -x
export ACTION={{ action }}
export BASE_DISTRO={{ base_distro }}
export INSTALL_TYPE={{ install_type }}
export IN_PROJECT_JOBS=1
tools/gate_run.sh
executable: /bin/bash
- name: Run tox
command: tox -e {{ action }}-{{ base_distro }}-{{ install_type }}
args:
chdir: "{{ zuul.project.src_dir }}"
environment: '{{ zuul | zuul_legacy_vars }}'

1
tests/setup_gate.sh
View File

@ -1 +0,0 @@
../tools/setup_gate.sh

2
tests/templates/kolla-build.conf.j2
View File

@ -5,5 +5,5 @@ namespace = kolla
tag = {{ zuul.tag if zuul.pipeline == "release" else zuul.branch | basename }}
{% endif %}
push = false
logs_dir = /tmp/logs/build
logs_dir = {{ kolla_build_logs_dir }}
template_override = /etc/kolla/template_overrides.j2

2
tools/dump_info.sh
View File

@ -23,4 +23,4 @@ function dump_node_info {
set -o errexit
}
(dump_node_info 2>&1) > /tmp/logs/node_info_$(date +%s)
(dump_node_info 2>&1) > node_info_$(date +%s)

16
tools/gate_run.sh
View File

@ -1,16 +0,0 @@
#!/bin/bash
set -o errexit
set -o xtrace
# exist when the jobs is not project jobs.
if [[ -z $IN_PROJECT_JOBS ]]; then
exit 0
fi
tools/setup_gate.sh
tox -e $ACTION-$BASE_DISTRO-$INSTALL_TYPE
if [[ -n $PACK_REGISTRY ]] && [[ $ACTION == "build" ]]; then
pack_registry
fi

15
tools/setup_gate.sh
View File

@ -1,15 +0,0 @@
#!/bin/bash
set -o xtrace
set -o errexit
# Enable unbuffered output for Ansible in Jenkins.
export PYTHONUNBUFFERED=1
function setup_registry {
sudo mkdir /opt/kolla_registry
sudo chmod -R 644 /opt/kolla_registry
docker run -d -p 4000:5000 --restart=always -v /opt/kolla_registry/:/var/lib/registry --name registry registry:2
}
setup_registry

13
tox.ini
View File

@ -75,11 +75,6 @@ commands =
doc8 doc
sphinx-build -W -b html doc/source doc/build/html
[testenv:setupenv]
commands =
{toxinidir}/tools/setup_gate.sh
{toxinidir}/tools/dump_info.sh
[testenv:build-centos-binary]
whitelist_externals = find
bash
@ -89,7 +84,6 @@ commands =
find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
bash -c "if [ ! -d .stestr ]; then stestr init; fi"
stestr run test_build.BuildTestCentosBinary
{toxinidir}/tools/dump_info.sh
[testenv:build-centos-source]
whitelist_externals = find
@ -100,7 +94,6 @@ commands =
find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
bash -c "if [ ! -d .stestr ]; then stestr init; fi"
stestr run test_build.BuildTestCentosSource
{toxinidir}/tools/dump_info.sh
[testenv:build-ubuntu-binary]
whitelist_externals = find
@ -111,7 +104,6 @@ commands =
find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
bash -c "if [ ! -d .stestr ]; then stestr init; fi"
stestr run test_build.BuildTestUbuntuBinary
{toxinidir}/tools/dump_info.sh
[testenv:build-ubuntu-source]
whitelist_externals = find
@ -122,7 +114,6 @@ commands =
find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
bash -c "if [ ! -d .stestr ]; then stestr init; fi"
stestr run test_build.BuildTestUbuntuSource
{toxinidir}/tools/dump_info.sh
[testenv:build-debian-binary]
whitelist_externals = find
@ -133,7 +124,6 @@ commands =
find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
bash -c "if [ ! -d .stestr ]; then stestr init; fi"
stestr run test_build.BuildTestDebianBinary
{toxinidir}/tools/dump_info.sh
[testenv:build-debian-source]
whitelist_externals = find
@ -144,7 +134,6 @@ commands =
find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
bash -c "if [ ! -d .stestr ]; then stestr init; fi"
stestr run test_build.BuildTestDebianSource
{toxinidir}/tools/dump_info.sh
[testenv:build-oraclelinux-binary]
whitelist_externals = find
@ -155,7 +144,6 @@ commands =
find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
bash -c "if [ ! -d .stestr ]; then stestr init; fi"
stestr run test_build.BuildTestOracleLinuxBinary
{toxinidir}/tools/dump_info.sh
[testenv:build-oraclelinux-source]
whitelist_externals = find
@ -166,7 +154,6 @@ commands =
find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
bash -c "if [ ! -d .stestr ]; then stestr init; fi"
stestr run test_build.BuildTestOracleLinuxSource
{toxinidir}/tools/dump_info.sh
[testenv:genconfig]
whitelist_externals = which