From 6f940da935218178b34172bf6bd1df59d6b8de8e Mon Sep 17 00:00:00 2001 From: Tristan Cacqueray Date: Thu, 12 Sep 2019 18:23:19 +0000 Subject: [PATCH] fetch-sphinx-tarball: introduce zuul_use_fetch_output This change enables using fetch-sphinx-tarball role along with the fetch-output role. By default the role still synchronizes artifacts back to the executor. Change-Id: I7452f34bccdca49e256053f9630f77953b27f99c --- roles/fetch-sphinx-tarball/README.rst | 8 +++ roles/fetch-sphinx-tarball/defaults/main.yaml | 2 + roles/fetch-sphinx-tarball/tasks/html.yaml | 61 +++++++++++++------ roles/fetch-sphinx-tarball/tasks/pdf.yaml | 45 ++++++++++---- .../python/fetch-sphinx-tarball.yaml | 45 ++++++++++++++ zuul-tests.d/python-jobs.yaml | 24 ++++++++ 6 files changed, 155 insertions(+), 30 deletions(-) create mode 100644 test-playbooks/python/fetch-sphinx-tarball.yaml diff --git a/roles/fetch-sphinx-tarball/README.rst b/roles/fetch-sphinx-tarball/README.rst index d78cf9fa9..0895ec894 100644 --- a/roles/fetch-sphinx-tarball/README.rst +++ b/roles/fetch-sphinx-tarball/README.rst @@ -23,3 +23,11 @@ archive into the log root for viewing. A list of file names of PDF files to collect. By default, the list contains as entry only ``doc-{{ zuul.project.short_name }}.pdf``. + +.. zuul:rolevar:: zuul_use_fetch_output + :default: false + + Whether to synchronize files to the executor work dir, or to copy them + on the test instance. + When set to false, the role synchronizes the file to the executor. + When set to true, the job needs to use the fetch-output role later. diff --git a/roles/fetch-sphinx-tarball/defaults/main.yaml b/roles/fetch-sphinx-tarball/defaults/main.yaml index a49d3c3cd..b9b0bb05e 100644 --- a/roles/fetch-sphinx-tarball/defaults/main.yaml +++ b/roles/fetch-sphinx-tarball/defaults/main.yaml @@ -1,5 +1,7 @@ --- zuul_work_dir: "{{ zuul.project.src_dir }}" +zuul_output_dir: "{{ ansible_user_dir }}/zuul-output" +zuul_use_fetch_output: "{{ zuul_site_use_fetch_output|default(false) }}" sphinx_build_dir: doc/build sphinx_pdf_files: - "doc-{{ zuul.project.short_name }}.pdf" diff --git a/roles/fetch-sphinx-tarball/tasks/html.yaml b/roles/fetch-sphinx-tarball/tasks/html.yaml index f0a58e8fa..e15609b0e 100644 --- a/roles/fetch-sphinx-tarball/tasks/html.yaml +++ b/roles/fetch-sphinx-tarball/tasks/html.yaml @@ -9,27 +9,50 @@ args: warn: false -- name: Fetch archive HTML - synchronize: - dest: "{{ zuul.executor.log_root }}/docs-html.tar.gz" - mode: pull - src: "{{ html_archive.path }}" - verify_host: true +- block: + - name: Fetch archive HTML + synchronize: + dest: "{{ zuul.executor.log_root }}/docs-html.tar.gz" + mode: pull + src: "{{ html_archive.path }}" + verify_host: true -- name: Create browseable HTML directory - delegate_to: localhost - file: - path: "{{ zuul.executor.log_root }}/docs" - state: directory + - name: Create browseable HTML directory + delegate_to: localhost + file: + path: "{{ zuul.executor.log_root }}/docs" + state: directory -- name: Extract archive HTML - delegate_to: localhost - unarchive: - src: "{{ zuul.executor.log_root }}/docs-html.tar.gz" - dest: "{{ zuul.executor.log_root }}/docs" - remote_src: true - extra_opts: - - "--no-same-owner" + - name: Extract archive HTML + delegate_to: localhost + unarchive: + src: "{{ zuul.executor.log_root }}/docs-html.tar.gz" + dest: "{{ zuul.executor.log_root }}/docs" + remote_src: true + extra_opts: + - "--no-same-owner" + when: not zuul_use_fetch_output + +- block: + - name: Copy archive HTML + copy: + dest: "{{ zuul_output_dir }}/logs/docs-html.tar.gz" + src: "{{ html_archive.path }}" + remote_src: true + + - name: Create browseable HTML directory + file: + path: "{{ zuul_output_dir }}/logs/docs" + state: directory + + - name: Extract archive HTML + unarchive: + src: "{{ zuul_output_dir }}/logs/docs-html.tar.gz" + dest: "{{ zuul_output_dir }}/logs/docs" + remote_src: true + extra_opts: + - "--no-same-owner" + when: zuul_use_fetch_output - name: Return artifact to Zuul zuul_return: diff --git a/roles/fetch-sphinx-tarball/tasks/pdf.yaml b/roles/fetch-sphinx-tarball/tasks/pdf.yaml index 2860fe624..086655f41 100644 --- a/roles/fetch-sphinx-tarball/tasks/pdf.yaml +++ b/roles/fetch-sphinx-tarball/tasks/pdf.yaml @@ -28,9 +28,10 @@ # Now loop... - name: Grab PDF files - when: pdf_files_found + when: + - pdf_files_found + - not zuul_use_fetch_output block: - - name: Create PDF directory delegate_to: localhost file: @@ -48,16 +49,38 @@ loop_var: zj_pdf when: zj_pdf.stat.exists - - name: Return PDF artifact to Zuul - zuul_return: - data: - zuul: - artifacts: - - name: "Docs PDF: {{ zj_pdf.zj_sphinx_pdf }}" - url: "pdf/{{ zj_pdf.zj_sphinx_pdf }}" - metadata: - type: docs_pdf +- name: Copy PDF files + when: + - pdf_files_found + - zuul_use_fetch_output + block: + - name: Create local PDF directory + file: + path: "{{ zuul_output_dir }}/logs/pdf" + state: directory + + - name: Copy PDF files + copy: + dest: "{{ zuul_output_dir }}/logs/pdf/{{ zj_pdf.zj_sphinx_pdf }}" + src: "{{ zj_pdf.stat.path }}" + remote_src: true with_items: "{{ pdf_file_stat.results }}" loop_control: loop_var: zj_pdf when: zj_pdf.stat.exists + +- name: Return PDF artifact to Zuul + zuul_return: + data: + zuul: + artifacts: + - name: "Docs PDF: {{ zj_pdf.zj_sphinx_pdf }}" + url: "pdf/{{ zj_pdf.zj_sphinx_pdf }}" + metadata: + type: docs_pdf + with_items: "{{ pdf_file_stat.results }}" + loop_control: + loop_var: zj_pdf + when: + - pdf_files_found + - zj_pdf.stat.exists diff --git a/test-playbooks/python/fetch-sphinx-tarball.yaml b/test-playbooks/python/fetch-sphinx-tarball.yaml new file mode 100644 index 000000000..5cd2a5832 --- /dev/null +++ b/test-playbooks/python/fetch-sphinx-tarball.yaml @@ -0,0 +1,45 @@ +- hosts: all + pre_tasks: + # Run ensure-output-dirs now as it is not performed speculatively + - import_role: + name: ensure-output-dirs + + - name: Create fake sphinx output + shell: | + mkdir -p {{ zuul.project.src_dir }}/doc/build/pdf + mkdir -p {{ zuul.project.src_dir }}/doc/build/html + echo "%PDF-1.2" > {{ zuul.project.src_dir }}/doc/build/pdf/doc-{{ zuul.project.short_name }}.pdf + echo "Hello" > {{ zuul.project.src_dir }}/doc/build/html/index.html + + tasks: + - import_role: + name: fetch-sphinx-tarball + + - import_role: + name: fetch-output + when: zuul_use_fetch_output + + - import_role: + name: merge-output-to-logs + when: zuul_use_fetch_output + + post_tasks: + - name: Check for artifact on the test instance + stat: + path: "{{ ansible_user_dir }}/zuul-output/logs/{{ item }}" + register: _test_artifact + failed_when: not _test_artifact.stat.exists + with_items: + - "pdf/doc-{{ zuul.project.short_name }}.pdf" + - docs/index.html + when: zuul_use_fetch_output + + - name: Check for artifact on the executor + stat: + path: "{{ zuul.executor.log_root }}/{{ item }}" + delegate_to: localhost + register: _executor_artifact + failed_when: not _executor_artifact.stat.exists + with_items: + - "pdf/doc-{{ zuul.project.short_name }}.pdf" + - docs/index.html diff --git a/zuul-tests.d/python-jobs.yaml b/zuul-tests.d/python-jobs.yaml index 8157ba7c1..9c6117485 100644 --- a/zuul-tests.d/python-jobs.yaml +++ b/zuul-tests.d/python-jobs.yaml @@ -478,6 +478,26 @@ vars: zuul_use_fetch_output: false +- job: + name: zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output + description: Test the fetch-sphinx-tarball + files: + - roles/ensure-output-dirs/.* + - roles/fetch-sphinx-tarball/.* + - roles/fetch-output/.* + run: test-playbooks/python/fetch-sphinx-tarball.yaml + vars: + zuul_use_fetch_output: true + +- job: + name: zuul-jobs-test-fetch-sphinx-tarball-synchronize + description: Test the fetch-sphinx-tarball + files: + - roles/fetch-sphinx-tarball/.* + run: test-playbooks/python/fetch-sphinx-tarball.yaml + vars: + zuul_use_fetch_output: false + - project: check: jobs: @@ -525,6 +545,8 @@ - zuul-jobs-test-fetch-subunit-output-synchronize - zuul-jobs-test-fetch-sphinx-output - zuul-jobs-test-fetch-sphinx-output-synchronize + - zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output + - zuul-jobs-test-fetch-sphinx-tarball-synchronize gate: jobs: - zuul-jobs-test-ensure-pip-centos-7 @@ -568,3 +590,5 @@ - zuul-jobs-test-fetch-subunit-output-synchronize - zuul-jobs-test-fetch-sphinx-output - zuul-jobs-test-fetch-sphinx-output-synchronize + - zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output + - zuul-jobs-test-fetch-sphinx-tarball-synchronize