Merge "Refactor fetch-sphinx-tarball to be executor safe"

doc/source/general-roles.rst

@@ -7,6 +7,7 @@ General Purpose Roles
 .. zuul:autorole:: add-sshkey
 .. zuul:autorole:: bindep
 .. zuul:autorole:: buildset-artifacts-location
+.. zuul:autorole:: clean-directory
 .. zuul:autorole:: clear-firewall
 .. zuul:autorole:: configure-mirrors
 .. zuul:autorole:: copy-build-sshkey

roles/clean-directory/README.rst

@@ -0,0 +1,10 @@
+Clean a directory, but leave the directory alone
+
+This is the equivalent of ``rm -rf *`` when run in a directory.  It is
+safe to run in executor context.
+
+**Role Variables**
+
+.. zuul:rolevar:: clean_directory_path
+
+   The directory to clean.

roles/clean-directory/tasks/main.yaml

@@ -0,0 +1,28 @@
+- name: Assert directory set
+  assert:
+    that: clean_directory_path is defined
+    fail_msg: 'Must specify clean_directory_path'
+
+- block:
+  - name: Collect files
+    find:
+      paths: '{{ clean_directory_path }}'
+      hidden: True
+      recurse: True
+    register: _files
+
+  - name: Collect directories
+    find:
+      paths: '{{ clean_directory_path }}'
+      hidden: True
+      recurse: True
+      file_type: directory
+    register: _dirs
+
+  - name: Clean Directory
+    file:
+      path: '{{ zj_item }}'
+      state: absent
+    loop: '{{ _files.files + _dirs.files }}'
+    loop_control:
+      loop_var: zj_item

roles/merge-output-to-logs/tasks/main.yaml

@@ -1,18 +1,45 @@
 - name: Move artifacts and docs to logs dir
   when: zuul.change is defined
   delegate_to: localhost
-  shell: |
-    if [ -n "$(find {{ zuul.executor.work_root }}/{{ zj_item }} -mindepth 1)" ] ; then
-      # Only create target directory if it is needed.
-      # Do not fail if it is already there.
-      mkdir -p {{ zuul.executor.log_root }}/{{ zj_item }}
-      # Leave the original directory behind so that other roles
-      # operating on the interface directories can simply no-op.
-      mv -f {{ zuul.executor.work_root }}/{{ zj_item }}/* {{ zuul.executor.log_root }}/{{ zj_item }}
-    fi
-  loop:
-    - artifacts
-    - docs
-  loop_control:
-    loop_var: zj_item
-  run_once: true
+  block:
+    # Artifacts
+    - name: Check if artifacts is empty
+      find:
+        paths: '{{ zuul.executor.work_root }}/artifacts'
+      register: _artifacts
+
+    - name: Copy artifacts into log root
+      synchronize:
+        src: '{{ zuul.executor.work_root }}/artifacts/'
+        dest: '{{ zuul.exector.log_root }}/artifacts/'
+        owner: no
+        group: no
+      when: _artifacts.files
+
+    - name: Clean artifacts
+      include_role:
+        name: clean-directory
+      vars:
+        clean_directory_path: '{{ zuul.executor.work_root }}/artifacts/'
+      when: _artifacts.files
+
+    # Docs
+    - name: Check if docs is empty
+      find:
+        paths: '{{ zuul.executor.work_root }}/docs'
+      register: _docs
+
+    - name: Copy docs into log root
+      synchronize:
+        src: '{{ zuul.executor.work_root }}/docs/'
+        dest: '{{ zuul.exector.log_root }}/docs/'
+        owner: no
+        group: no
+      when: _docs.files
+
+    - name: Clean docs
+      include_role:
+        name: clean-directory
+      vars:
+        clean_directory_path: '{{ zuul.executor.work_root }}/docs/'
+      when: _docs.files

zuul-tests.d/python-jobs.yaml

@@ -455,19 +455,19 @@
     vars:
       zuul_use_fetch_output: false
 
-# Disabled because "Move artifacts and docs to logs dir" task would
-# fail with "Executing local code is prohibited" during testing
-# - job:
-#     name: zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output
-#     description: Test the fetch-sphinx-tarball
-#     files:
-#       - roles/ensure-output-dirs/.*
-#       - roles/fetch-sphinx-tarball/.*
-#       - roles/fetch-output/.*
-#     run: test-playbooks/python/fetch-sphinx-tarball.yaml
-#     voting: false
-#     vars:
-#       zuul_use_fetch_output: true
+- job:
+    name: zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output
+    description: Test the fetch-sphinx-tarball
+    files:
+      - roles/ensure-output-dirs/.*
+      - roles/fetch-sphinx-tarball/.*
+      - roles/fetch-output/.*
+      - roles/merge-output-to-logs/.*
+      - roles/clean-directory/.*
+      - test-playbooks/python/fetch-sphinx-tarball.yaml
+    run: test-playbooks/python/fetch-sphinx-tarball.yaml
+    vars:
+      zuul_use_fetch_output: true
 
 - job:
     name: zuul-jobs-test-fetch-sphinx-tarball-synchronize
@@ -522,6 +522,7 @@
         - zuul-jobs-test-fetch-subunit-output-synchronize
         - zuul-jobs-test-fetch-sphinx-output
         - zuul-jobs-test-fetch-sphinx-output-synchronize
+        - zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output
         - zuul-jobs-test-fetch-sphinx-tarball-synchronize
     gate:
       jobs: &id001
@@ -563,6 +564,7 @@
         - zuul-jobs-test-fetch-subunit-output-synchronize
         - zuul-jobs-test-fetch-sphinx-output
         - zuul-jobs-test-fetch-sphinx-output-synchronize
+        - zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output
         - zuul-jobs-test-fetch-sphinx-tarball-synchronize
     periodic-weekly:
       jobs: *id001