From c7ac0f0de0c65afe54974a7ea727feb27c0f60eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Tue, 11 Jun 2019 15:10:13 +0200 Subject: [PATCH] Clean up test process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If stestr failed, then no node info was logged. Logging node info is now handled in pre and post stages of Zuul. Additionally Zuul v3 migration recommendations were applied and some unused code removed improving general readability. Change-Id: Iae9d37cb76997ba8389e21c7e2a3bf5905abe029 Signed-off-by: Radosław Piliszek --- .zuul.d/base.yaml | 6 +-- tests/playbooks/post.yml | 83 ++++++++++++++--------------- tests/playbooks/pre.yml | 22 ++++---- tests/playbooks/run.yml | 35 ++---------- tests/setup_gate.sh | 1 - tests/templates/kolla-build.conf.j2 | 2 +- tools/dump_info.sh | 2 +- tools/gate_run.sh | 16 ------ tools/setup_gate.sh | 15 ------ tox.ini | 13 ----- 10 files changed, 62 insertions(+), 133 deletions(-) delete mode 120000 tests/setup_gate.sh delete mode 100755 tools/gate_run.sh delete mode 100755 tools/setup_gate.sh diff --git a/.zuul.d/base.yaml b/.zuul.d/base.yaml index 0e2fbbec50..2fc3dcf081 100644 --- a/.zuul.d/base.yaml +++ b/.zuul.d/base.yaml @@ -46,9 +46,6 @@ run: tests/playbooks/run.yml post-run: tests/playbooks/post.yml attempts: 1 - required-projects: - - openstack/kolla-ansible - - openstack/requirements irrelevant-files: - ^.*\.rst$ - ^doc/.* @@ -59,3 +56,6 @@ - ^test-requirements.txt$ vars: publisher: false + extra-vars: + kolla_logs_dir: "{{ zuul_output_dir }}/logs/kolla" + kolla_build_logs_dir: "{{ kolla_logs_dir }}/build" diff --git a/tests/playbooks/post.yml b/tests/playbooks/post.yml index bb514821d8..00f71f6b9b 100644 --- a/tests/playbooks/post.yml +++ b/tests/playbooks/post.yml @@ -1,60 +1,66 @@ --- - hosts: all tasks: - - shell: + - name: Dump host info to logs + command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" + args: + chdir: "{{ kolla_logs_dir }}" + + - name: Collect various info to logs + shell: cmd: | set +o errexit - mkdir -p logs - # copy system log + set -o xtrace - sudo cp -r /var/log logs/system_log + # copy system log + sudo cp -r /var/log system_log if which journalctl ; then # the journal gives us syslog() and kernel output, so is like # a concatenation of the above. - sudo journalctl --no-pager | sudo tee logs/syslog.txt > /dev/null - sudo journalctl --no-pager -u docker.service | sudo tee logs/docker.log > /dev/null + sudo journalctl --no-pager > syslog.txt + sudo journalctl --no-pager -u docker.service > docker.log else # assume rsyslog - sudo cp /var/log/syslog logs/syslog.txt - sudo cp /var/log/kern.log logs/kern_log.txt - sudo cp /var/log/upstart/docker.log logs/docker.log + sudo cp /var/log/syslog syslog.txt + sudo cp /var/log/kern.log kern_log.txt + sudo cp /var/log/upstart/docker.log docker.log fi if sudo test -d /var/lib/docker/volumes/kolla_logs/_data; then - sudo cp -r /var/lib/docker/volumes/kolla_logs/_data logs/kolla_logs + sudo cp -r /var/lib/docker/volumes/kolla_logs/_data kolla_logs fi # sudo config - sudo cp -r /etc/sudoers.d logs/ - sudo cp /etc/sudoers logs/sudoers.txt + sudo cp -r /etc/sudoers.d ./ + sudo cp /etc/sudoers sudoers.txt - df -h > logs/df.txt - free > logs/free.txt - sudo parted -l | sudo tee logs/parted-l.txt > /dev/null - mount > logs/mount.txt - env > logs/env.txt + df -h > df.txt + free > free.txt + sudo parted -l > parted-l.txt + mount > mount.txt + env > env.txt if [ `command -v dpkg` ]; then - dpkg -l | sudo tee logs/dpkg-l.txt > /dev/null + dpkg -l > dpkg-l.txt fi if [ `command -v rpm` ]; then - rpm -qa | sudo tee logs/rpm-qa.txt > /dev/null + rpm -qa > rpm-qa.txt fi # final memory usage and process list - ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > logs/ps.txt + ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ps.txt # docker related information - (docker info && docker system df && docker system df -v) > logs/docker-info.txt + (docker info && docker system df && docker system df -v) > docker-info.txt - sudo cp -r /etc/kolla logs/kolla_configs + sudo cp -r /etc/kolla kolla_configs - # fix the permissions for logs folder - sudo chmod -R 777 logs + # last sudo to own all files + sudo chown -R {{ ansible_user_id }}: . # Parse build logs and extract pkg install info - tests/files/process_build_logs.py -l logs/build -b {{ base_distro }} > logs/packages-info.txt + {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py -l build -b {{ base_distro }} > packages-info.txt # rename files to .txt; this is so that when displayed via # logs.openstack.org clicking results in the browser shows the @@ -62,30 +68,21 @@ # download it, etc. # firstly, rename all .log files to .txt files - for f in $(find logs -name "*.log"); do - sudo mv $f ${f/.log/.txt} + for f in $(find . -name "*.log"); do + mv $f ${f/.log/.txt} done # Update symlinks to new file names - for f in $(find logs -name "*FAILED*"); do - sudo mv ${f} ${f}.gz - sudo ln -sf ${f#*/000_FAILED_}.gz ${f}.gz + for f in $(find . -name "*FAILED*"); do + mv ${f} ${f}.gz + ln -sf ${f#*/000_FAILED_}.gz ${f}.gz done # append .txt to all kolla config file - find logs/kolla_configs -type f -exec mv '{}' '{}'.txt \; + find kolla_configs -type f -exec mv '{}' '{}'.txt \; # Compress all text logs - find logs -iname '*.txt' -execdir gzip -9 {} \+ - find logs -iname '*.json' -execdir gzip -9 {} \+ + find . -iname '*.txt' -execdir gzip -9 {} \+ + find . -iname '*.json' -execdir gzip -9 {} \+ executable: /bin/bash - chdir: "{{ zuul.project.src_dir }}" - - - synchronize: - src: '{{ zuul.project.src_dir }}/logs' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - "--quiet" + chdir: "{{ kolla_logs_dir }}" diff --git a/tests/playbooks/pre.yml b/tests/playbooks/pre.yml index 00d471614c..e721d88592 100644 --- a/tests/playbooks/pre.yml +++ b/tests/playbooks/pre.yml @@ -1,13 +1,17 @@ --- - hosts: all - become: true tasks: - - name: Install dbus for debian system - apt: name=dbus - when: ansible_os_family == 'Debian' + - name: Create dir for kolla logs + file: + path: "{{ kolla_logs_dir }}" + state: directory - - name: Install ansible - pip: - name: ansible - when: - - inventory_hostname == "primary" + - name: Dump host info to logs + command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh" + args: + chdir: "{{ kolla_logs_dir }}" + + - name: Create dir for kolla build logs + file: + path: "{{ kolla_build_logs_dir }}" + state: directory diff --git a/tests/playbooks/run.yml b/tests/playbooks/run.yml index b15ec45b96..0578dea32f 100644 --- a/tests/playbooks/run.yml +++ b/tests/playbooks/run.yml @@ -1,7 +1,6 @@ --- - hosts: all vars: - kolla_inventory_path: "{{ zuul.project.src_dir }}/inventory" nodepool_rdo_proxy: "{{ zuul_site_mirror_fqdn }}:8080/rdo" nodepool_pypi_mirror: "http://{{ zuul_site_mirror_fqdn }}:8080/pypi/simple" nodepool_mirror_host: "{{ zuul_site_mirror_fqdn }}" @@ -12,23 +11,6 @@ nodepool_oraclelinux_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/oraclelinux/" nodepool_percona_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/percona" tasks: - - name: generate kolla inventory file - copy: - dest: "{{ kolla_inventory_path }}" - content: | - {% for host in hostvars %} - {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user={{ hostvars[host]['ansible_user'] }} - {% endfor %} - - - name: setup logging - shell: - cmd: | - mkdir logs - ln -s $(pwd)/logs /tmp/logs - mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs} - executable: /bin/bash - chdir: "{{ zuul.project.src_dir }}" - - name: copy setup script copy: src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tools/setup_{{ ansible_os_family }}.sh" @@ -57,21 +39,12 @@ src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2" dest: /etc/kolla/kolla-build.conf - - name: Template override + - name: Template template_overrides.j2 template: src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/template_overrides.j2" dest: /etc/kolla/template_overrides.j2 - - shell: - cmd: | - set -e - set -x - - export ACTION={{ action }} - export BASE_DISTRO={{ base_distro }} - export INSTALL_TYPE={{ install_type }} - export IN_PROJECT_JOBS=1 - tools/gate_run.sh - executable: /bin/bash + - name: Run tox + command: tox -e {{ action }}-{{ base_distro }}-{{ install_type }} + args: chdir: "{{ zuul.project.src_dir }}" - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh deleted file mode 120000 index fbacb3f920..0000000000 --- a/tests/setup_gate.sh +++ /dev/null @@ -1 +0,0 @@ -../tools/setup_gate.sh \ No newline at end of file diff --git a/tests/templates/kolla-build.conf.j2 b/tests/templates/kolla-build.conf.j2 index 3f9e5d04ce..37b06aca68 100644 --- a/tests/templates/kolla-build.conf.j2 +++ b/tests/templates/kolla-build.conf.j2 @@ -5,5 +5,5 @@ namespace = kolla tag = {{ zuul.tag if zuul.pipeline == "release" else zuul.branch | basename }} {% endif %} push = false -logs_dir = /tmp/logs/build +logs_dir = {{ kolla_build_logs_dir }} template_override = /etc/kolla/template_overrides.j2 diff --git a/tools/dump_info.sh b/tools/dump_info.sh index b0b3ce3f1e..26158653bc 100755 --- a/tools/dump_info.sh +++ b/tools/dump_info.sh @@ -23,4 +23,4 @@ function dump_node_info { set -o errexit } -(dump_node_info 2>&1) > /tmp/logs/node_info_$(date +%s) +(dump_node_info 2>&1) > node_info_$(date +%s) diff --git a/tools/gate_run.sh b/tools/gate_run.sh deleted file mode 100755 index 07f5e35743..0000000000 --- a/tools/gate_run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o xtrace - -# exist when the jobs is not project jobs. -if [[ -z $IN_PROJECT_JOBS ]]; then - exit 0 -fi - -tools/setup_gate.sh -tox -e $ACTION-$BASE_DISTRO-$INSTALL_TYPE - -if [[ -n $PACK_REGISTRY ]] && [[ $ACTION == "build" ]]; then - pack_registry -fi diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh deleted file mode 100755 index e2c617dd0e..0000000000 --- a/tools/setup_gate.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -o xtrace -set -o errexit - -# Enable unbuffered output for Ansible in Jenkins. -export PYTHONUNBUFFERED=1 - -function setup_registry { - sudo mkdir /opt/kolla_registry - sudo chmod -R 644 /opt/kolla_registry - docker run -d -p 4000:5000 --restart=always -v /opt/kolla_registry/:/var/lib/registry --name registry registry:2 -} - -setup_registry diff --git a/tox.ini b/tox.ini index 5d4e9902dd..a448970ac5 100644 --- a/tox.ini +++ b/tox.ini @@ -75,11 +75,6 @@ commands = doc8 doc sphinx-build -W -b html doc/source doc/build/html -[testenv:setupenv] -commands = - {toxinidir}/tools/setup_gate.sh - {toxinidir}/tools/dump_info.sh - [testenv:build-centos-binary] whitelist_externals = find bash @@ -89,7 +84,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestCentosBinary - {toxinidir}/tools/dump_info.sh [testenv:build-centos-source] whitelist_externals = find @@ -100,7 +94,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestCentosSource - {toxinidir}/tools/dump_info.sh [testenv:build-ubuntu-binary] whitelist_externals = find @@ -111,7 +104,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestUbuntuBinary - {toxinidir}/tools/dump_info.sh [testenv:build-ubuntu-source] whitelist_externals = find @@ -122,7 +114,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestUbuntuSource - {toxinidir}/tools/dump_info.sh [testenv:build-debian-binary] whitelist_externals = find @@ -133,7 +124,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestDebianBinary - {toxinidir}/tools/dump_info.sh [testenv:build-debian-source] whitelist_externals = find @@ -144,7 +134,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestDebianSource - {toxinidir}/tools/dump_info.sh [testenv:build-oraclelinux-binary] whitelist_externals = find @@ -155,7 +144,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestOracleLinuxBinary - {toxinidir}/tools/dump_info.sh [testenv:build-oraclelinux-source] whitelist_externals = find @@ -166,7 +154,6 @@ commands = find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete bash -c "if [ ! -d .stestr ]; then stestr init; fi" stestr run test_build.BuildTestOracleLinuxSource - {toxinidir}/tools/dump_info.sh [testenv:genconfig] whitelist_externals = which