Publish tarballs artifacts to AFS in parallel

This adds publishing of artifacts to AFS in parallel with the existing
publishing to the static volume.

The additional tasks added to the openstack-artifacts.yaml playbook
have been tested in the project-config-test-artifacts-to-afs test
(results at [1]) and also in an external project
ironic-python-agent-builder (see notes in [2]).  Thus we can have high
confidence it will work.

After this merges, I will manually rsync the old static tarballs data
to the AFS directory (again; it has been sync'd previously) and from
that point on both should be exactly the same.

At that point we can switch the DNS entries for
static.<openstack|opendev>.org (the sites have been created with [3])
to the new server and we will be fully migrated to the AFS volume.  We
can then remove the static publishing.

[1] http://files.openstack.org/project/tarballs.opendev.org/openstack/project-config/afs-copy-test/
[2] https://review.opendev.org/705161
[3] https://review.opendev.org/704913

Change-Id: Ia3a0358249e9ed3d766b1b61535f2f6d67d4eb2d

playbooks/publish/openstack-artifacts-with-afs-test.yaml

@@ -1,15 +0,0 @@
-- hosts: all
-
-  tasks:
-
-    - name: Ensure artifacts directory exists
-      file:
-        path: '{{ zuul.executor.work_root }}/artifacts/afs-copy-test/'
-        state: directory
-      delegate_to: localhost
-
-    - name: Write a dummy artifact file
-      copy:
-        content: 'This is an artifact created by {{ zuul.change }}/{{ zuul.patchset }}'
-        dest: '{{ zuul.executor.work_root }}/artifacts/afs-copy-test/{{ zuul.change }}.txt'
-      delegate_to: localhost

playbooks/publish/openstack-artifacts-with-afs.yaml

@@ -1,16 +0,0 @@
-- hosts: localhost
-  tasks:
-    - block:
-        - name: Create AFS token
-          include_role:
-            name: create-afs-token
-        - name: Upload to afs
-          include_role:
-            name: upload-afs-synchronize
-          vars:
-            afs_source: "{{ zuul.executor.work_root }}/artifacts/"
-            afs_target: "/afs/.openstack.org/project/tarballs.opendev.org/{{ zuul.project.name}}/"
-        - name: Destroy AFS token
-          include_role:
-            name: destroy-afs-token
-      when: zuul_success | bool

playbooks/publish/openstack-artifacts.yaml

@@ -8,3 +8,23 @@
   roles:
     - role: publish-artifacts-to-fileserver
       when: zuul_success | bool
+
+# NOTE(ianw) 2020-02-10 : upload to AFS in parallel until we switch
+# DNS for the static.<openstack|opendev>.org servers.  After that
+# remove above section to stop static server publishing.
+- hosts: localhost
+  tasks:
+    - block:
+        - name: Create AFS token
+          include_role:
+            name: create-afs-token
+        - name: Upload to afs
+          include_role:
+            name: upload-afs-synchronize
+          vars:
+            afs_source: "{{ zuul.executor.work_root }}/artifacts/"
+            afs_target: "/afs/.openstack.org/project/tarballs.opendev.org/{{ zuul.project.name}}/"
+        - name: Destroy AFS token
+          include_role:
+            name: destroy-afs-token
+      when: zuul_success | bool

zuul.d/jobs.yaml

@@ -78,17 +78,6 @@
     secrets:
       - secret: site_tarballs
         name: fileserver
-
-# NOTE(ianw) 2020-01-30 : we are moving tarballs to publish to an AFS
-# volume.  This job is intended to replace publish-openstack-artifacts
-# when it is debugged and ready.
-- job:
-    name: publish-openstack-artifacts-with-afs
-    description: |
-      Publish job to upload artifacts to /afs/.openstack.org/project/tarballs.opendev.org
-    post-review: true
-    post-run: playbooks/publish/openstack-artifacts-with-afs.yaml
-    secrets:
       - secret: afs_tarballs_opendev_org
         name: afs
 
@@ -1188,11 +1177,3 @@
 
       The testing is done for Ubuntu Xenial.
     nodeset: ubuntu-xenial
-
-# NOTE(ianw) : 2020-01-31 temporary job for testing
-- job:
-    name: project-config-test-artifacts-to-afs
-    parent: publish-openstack-artifacts-with-afs
-    description: |
-      Temporary job to test uploading artifacts to AFS
-    run: playbooks/publish/openstack-artifacts-with-afs-test.yaml

zuul.d/projects.yaml

@@ -3274,7 +3274,6 @@
             files: *infra-docs-index
         - promote-openstack-specs-site:
             files: *specs-site
-        - project-config-test-artifacts-to-afs
     periodic:
       jobs:
         - propose-project-config-update