From 56ee80234cae7336d2857f88b6f130a499c0c8eb Mon Sep 17 00:00:00 2001 From: Tristan Cacqueray Date: Thu, 12 Sep 2019 17:58:48 +0000 Subject: [PATCH] fetch-tox-output: introduce zuul_use_fetch_output This change enables using fetch-tox role along with the fetch-output role. By default the role still synchronizes artifacts back to the executor. Change-Id: Iffe5ffc52ee4c765b2e23ab8b3235b6bc7c510d8 --- roles/fetch-tox-output/README.rst | 8 +++ roles/fetch-tox-output/defaults/main.yaml | 3 + roles/fetch-tox-output/tasks/main.yaml | 18 ++++++ test-playbooks/python/fetch-tox-output.yaml | 67 +++++++++++++++++++++ zuul-tests.d/python-jobs.yaml | 22 +++++++ 5 files changed, 118 insertions(+) create mode 100644 test-playbooks/python/fetch-tox-output.yaml diff --git a/roles/fetch-tox-output/README.rst b/roles/fetch-tox-output/README.rst index 7fe6f1219..a20fb1f4d 100644 --- a/roles/fetch-tox-output/README.rst +++ b/roles/fetch-tox-output/README.rst @@ -16,3 +16,11 @@ Collect log output from a tox build :default: {{ zuul.project.src_dir }} Directory tox was run in. + +.. zuul:rolevar:: zuul_use_fetch_output + :default: false + + Whether to synchronize files to the executor work dir, or to copy them + on the test instance. + When set to false, the role synchronizes the file to the executor. + When set to true, the job needs to use the fetch-output role later. diff --git a/roles/fetch-tox-output/defaults/main.yaml b/roles/fetch-tox-output/defaults/main.yaml index dddc1c877..cce0be99e 100644 --- a/roles/fetch-tox-output/defaults/main.yaml +++ b/roles/fetch-tox-output/defaults/main.yaml @@ -5,3 +5,6 @@ tox_envlist: venv tox_executable: tox zuul_work_dir: "{{ zuul.project.src_dir }}" + +zuul_output_dir: "{{ ansible_user_dir }}/zuul-output" +zuul_use_fetch_output: "{{ zuul_site_use_fetch_output|default(false) }}" diff --git a/roles/fetch-tox-output/tasks/main.yaml b/roles/fetch-tox-output/tasks/main.yaml index 2fe05359e..c6540cd16 100644 --- a/roles/fetch-tox-output/tasks/main.yaml +++ b/roles/fetch-tox-output/tasks/main.yaml @@ -14,6 +14,12 @@ state: directory delegate_to: localhost +- name: Ensure zuul-output tox dir + file: + path: "{{ zuul_output_dir }}/logs/tox" + state: directory + when: zuul_use_fetch_output + - name: Set envlist fact set_fact: envlist: "{{ tox_envlist.split(',') }}" @@ -31,6 +37,17 @@ envlist: "{{ tox_environments.stdout_lines }}" when: tox_envlist is not defined or tox_envlist|lower == 'all' +- name: Copy tox logs + copy: + dest: "{{ zuul_output_dir }}/logs/tox/" + src: "{{ zuul_work_dir }}/.tox/{{ item }}/log/" + remote_src: true + with_items: "{{ envlist }}" + # some tox runs may not create a virtualenv and thus have + # no ./tox/env directory + failed_when: false + when: zuul_use_fetch_output + - name: Collect tox logs synchronize: dest: "{{ log_path }}" @@ -41,3 +58,4 @@ # some tox runs may not create a virtualenv and thus have # no ./tox/env directory failed_when: false + when: not zuul_use_fetch_output diff --git a/test-playbooks/python/fetch-tox-output.yaml b/test-playbooks/python/fetch-tox-output.yaml new file mode 100644 index 000000000..9812fb185 --- /dev/null +++ b/test-playbooks/python/fetch-tox-output.yaml @@ -0,0 +1,67 @@ +- hosts: all + pre_tasks: + # Run ensure-output-dirs now as it is not performed speculatively + - import_role: + name: ensure-output-dirs + + - name: Simplify tox config + copy: + content: "{{ item.content }}" + dest: "{{ zuul.project.src_dir }}/{{ item.dest }}" + with_items: + - content: | + [testenv] + sitepackages = True + usedevelop = True + + [testenv:venv] + commands = python -c 'exit(0)' + dest: tox.ini + - content: | + import setuptools + setuptools.setup() + dest: setup.py + - content: "" + dest: setup.cfg + + - name: Generate tox results + include_role: + name: "{{ item }}" + with_items: + - ensure-tox + - ensure-python + - tox + + tasks: + - import_role: + name: fetch-tox-output + + - block: + - name: Undo the log_path fact set by fetch-javascript-output + set_fact: + log_path: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + when: groups['all'] | length > 1 + + - name: Undo the log_path fact set log path for single node + set_fact: + log_path: "{{ zuul.executor.log_root }}" + when: groups['all'] | length == 1 + + - import_role: + name: fetch-output + when: zuul_use_fetch_output + + post_tasks: + - name: Check for artifact on the test instance + stat: + path: "{{ ansible_user_dir }}/zuul-output/logs/tox/venv-0.log" + register: _test_artifact + failed_when: not _test_artifact.stat.exists + when: zuul_use_fetch_output + + - name: Check for artifact on the executor + stat: + path: "{{ zuul.executor.log_root }}/tox/venv-0.log" + delegate_to: localhost + register: _executor_artifact + failed_when: not _executor_artifact.stat.exists diff --git a/zuul-tests.d/python-jobs.yaml b/zuul-tests.d/python-jobs.yaml index 79f987c26..e2dafeb0b 100644 --- a/zuul-tests.d/python-jobs.yaml +++ b/zuul-tests.d/python-jobs.yaml @@ -22,10 +22,32 @@ # tools/update-test-platforms.py # Please re-run to generate new job lists +- job: + name: zuul-jobs-test-fetch-tox-output + description: Test the fetch-tox-output + files: + - roles/ensure-output-dirs/.* + - roles/fetch-tox-output/.* + - roles/fetch-output/.* + run: test-playbooks/python/fetch-tox-output.yaml + vars: + zuul_use_fetch_output: true + +- job: + name: zuul-jobs-test-fetch-tox-output-synchronize + description: Test the fetch-tox-output + files: + - roles/fetch-tox-output/.* + run: test-playbooks/python/fetch-tox-output.yaml + vars: + zuul_use_fetch_output: false + - project: check: jobs: &id001 - zuul-jobs-test-ensure-sphinx - zuul-jobs-test-tox-siblings + - zuul-jobs-test-fetch-tox-output + - zuul-jobs-test-fetch-tox-output-synchronize gate: jobs: *id001