From cd258e5a1d75db71c39b07c21f4c14b8a3926e5e Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 3 Feb 2020 11:09:18 +1100 Subject: [PATCH] upload-afs: rename to upload-afs-roots; add afs-upload-synchronize The upload-afs role is actually fairly specialised for uploading docs with its root-marker scheme. As we're moving more parts of our infrastructure to publish via AFS I think it's worth making this quite explicit by renaming the upload-afs role to upload-afs-roots. This should be transparent as we call the renamed role, and once callers have been migrated we can use the usual deprecation process. Add a partner role, upload-afs-synchronize, which is more of a straight copy with options to synchronize. This will be suitable for uploading tarball and release artifacts. This is similar in concept to the promotion job used by zuul [1], but in this case will be focused on the extant publishing jobs in project-config which don't currently use the artifact-reporting/promote process. [1] https://opendev.org/opendev/base-jobs/src/branch/master/playbooks/artifacts/promote.yaml#L39 Change-Id: I493d8829e3fd98e84f1b8f0e776e7ba41abf16c2 Story: #2006598 Task: #38600 --- doc/source/afs-roles.rst | 2 ++ roles/upload-afs-roots/README.rst | 26 +++++++++++++++++++ .../__init__.py | 0 .../defaults/main.yaml | 0 .../library/__init__.py | 0 .../library/zuul_afs.py | 0 roles/upload-afs-roots/tasks/main.yaml | 4 +++ roles/upload-afs-synchronize/README.rst | 19 ++++++++++++++ .../upload-afs-synchronize/defaults/main.yaml | 2 ++ roles/upload-afs-synchronize/tasks/main.yaml | 12 +++++++++ roles/upload-afs/README.rst | 25 +----------------- roles/upload-afs/tasks/main.yaml | 7 +++-- 12 files changed, 69 insertions(+), 28 deletions(-) create mode 100644 roles/upload-afs-roots/README.rst rename roles/{upload-afs => upload-afs-roots}/__init__.py (100%) rename roles/{upload-afs => upload-afs-roots}/defaults/main.yaml (100%) rename roles/{upload-afs => upload-afs-roots}/library/__init__.py (100%) rename roles/{upload-afs => upload-afs-roots}/library/zuul_afs.py (100%) create mode 100644 roles/upload-afs-roots/tasks/main.yaml create mode 100644 roles/upload-afs-synchronize/README.rst create mode 100644 roles/upload-afs-synchronize/defaults/main.yaml create mode 100644 roles/upload-afs-synchronize/tasks/main.yaml diff --git a/doc/source/afs-roles.rst b/doc/source/afs-roles.rst index c2e15e64d..bb67106bd 100644 --- a/doc/source/afs-roles.rst +++ b/doc/source/afs-roles.rst @@ -3,5 +3,7 @@ AFS Roles .. zuul:autorole:: destroy-afs-token .. zuul:autorole:: upload-afs +.. zuul:autorole:: upload-afs-roots +.. zuul:autorole:: upload-afs-synchronize .. zuul:autorole:: create-afs-token .. zuul:autorole:: release-afs-volume diff --git a/roles/upload-afs-roots/README.rst b/roles/upload-afs-roots/README.rst new file mode 100644 index 000000000..819f38174 --- /dev/null +++ b/roles/upload-afs-roots/README.rst @@ -0,0 +1,26 @@ +Copy contents from ``{{ zuul.executor.work_root }}/artifacts/`` to AFS + +This is intented for documentation publishing, it deletes files that +do not exist in the content from the source. + +Before the job rsyncs the build into its final location, it must first +create a list of directories that should not be deleted. This way if +an entire directory is removed from a document, it will still be +removed from the website, but directories which are themselves roots +of other documents (for example, the stein branch) are not removed. A +marker file, called `.root-marker`, at the root of each such directory +will accomplish this; therefore each build job should also ensure that +it leaves such a marker file at the root of its build. The job will +find each of those in the destination hierarchy and add their +containing directories to a list of directories to exclude from +rsyncing. + +**Role Variables** + +.. zuul:rolevar:: afs_source + + Path to local source directory. + +.. zuul:rolevar:: afs_target + + Target path in AFS (should begin with '/afs/...'). diff --git a/roles/upload-afs/__init__.py b/roles/upload-afs-roots/__init__.py similarity index 100% rename from roles/upload-afs/__init__.py rename to roles/upload-afs-roots/__init__.py diff --git a/roles/upload-afs/defaults/main.yaml b/roles/upload-afs-roots/defaults/main.yaml similarity index 100% rename from roles/upload-afs/defaults/main.yaml rename to roles/upload-afs-roots/defaults/main.yaml diff --git a/roles/upload-afs/library/__init__.py b/roles/upload-afs-roots/library/__init__.py similarity index 100% rename from roles/upload-afs/library/__init__.py rename to roles/upload-afs-roots/library/__init__.py diff --git a/roles/upload-afs/library/zuul_afs.py b/roles/upload-afs-roots/library/zuul_afs.py similarity index 100% rename from roles/upload-afs/library/zuul_afs.py rename to roles/upload-afs-roots/library/zuul_afs.py diff --git a/roles/upload-afs-roots/tasks/main.yaml b/roles/upload-afs-roots/tasks/main.yaml new file mode 100644 index 000000000..3b01008f5 --- /dev/null +++ b/roles/upload-afs-roots/tasks/main.yaml @@ -0,0 +1,4 @@ +- name: Synchronize files to AFS + zuul_afs: + source: "{{ afs_source }}" + target: "{{ afs_target }}" diff --git a/roles/upload-afs-synchronize/README.rst b/roles/upload-afs-synchronize/README.rst new file mode 100644 index 000000000..c2c902f1e --- /dev/null +++ b/roles/upload-afs-synchronize/README.rst @@ -0,0 +1,19 @@ +Copy contents from ``{{ zuul.executor.work_root }}/artifacts/`` to AFS + +**Role Variables** + +.. zuul:rolevar:: afs_source + + Path to local source directory. + +.. zuul:rolevar:: afs_target + + Target path in AFS (should begin with '/afs/...'). + +.. zuul:rolevar:: afs_copy_only + :default: True + + If set to `false`, this will specify `--delete-after` to remove + files on the remote side that do not exist on the copying side. + When set to `true` will act as a regular additive copy process and + will not remove any remote files. diff --git a/roles/upload-afs-synchronize/defaults/main.yaml b/roles/upload-afs-synchronize/defaults/main.yaml new file mode 100644 index 000000000..0fd7b56ee --- /dev/null +++ b/roles/upload-afs-synchronize/defaults/main.yaml @@ -0,0 +1,2 @@ +afs_source: "{{ zuul.executor.work_root }}/artifacts/" +afs_copy_only: true diff --git a/roles/upload-afs-synchronize/tasks/main.yaml b/roles/upload-afs-synchronize/tasks/main.yaml new file mode 100644 index 000000000..7f04796d1 --- /dev/null +++ b/roles/upload-afs-synchronize/tasks/main.yaml @@ -0,0 +1,12 @@ +- name: Upload contents to AFS + synchronize: + src: "{{ afs_source }}" + dest: "{{ afs_target }}" + # NOTE(ianw): you can't set group permissions on AFS, hence we + # don't set owner specifically. + archive: false + perms: true + times: true + recursive: true + rsync_opts: '{{ ["--safe-links"] + ["--delete-after"] if not afs_copy_only else [] }}' + diff --git a/roles/upload-afs/README.rst b/roles/upload-afs/README.rst index 819f38174..068eb5285 100644 --- a/roles/upload-afs/README.rst +++ b/roles/upload-afs/README.rst @@ -1,26 +1,3 @@ Copy contents from ``{{ zuul.executor.work_root }}/artifacts/`` to AFS -This is intented for documentation publishing, it deletes files that -do not exist in the content from the source. - -Before the job rsyncs the build into its final location, it must first -create a list of directories that should not be deleted. This way if -an entire directory is removed from a document, it will still be -removed from the website, but directories which are themselves roots -of other documents (for example, the stein branch) are not removed. A -marker file, called `.root-marker`, at the root of each such directory -will accomplish this; therefore each build job should also ensure that -it leaves such a marker file at the root of its build. The job will -find each of those in the destination hierarchy and add their -containing directories to a list of directories to exclude from -rsyncing. - -**Role Variables** - -.. zuul:rolevar:: afs_source - - Path to local source directory. - -.. zuul:rolevar:: afs_target - - Target path in AFS (should begin with '/afs/...'). +This role is deprecated for :zuul:role:`upload-afs-roots` diff --git a/roles/upload-afs/tasks/main.yaml b/roles/upload-afs/tasks/main.yaml index 3b01008f5..454327c47 100644 --- a/roles/upload-afs/tasks/main.yaml +++ b/roles/upload-afs/tasks/main.yaml @@ -1,4 +1,3 @@ -- name: Synchronize files to AFS - zuul_afs: - source: "{{ afs_source }}" - target: "{{ afs_target }}" +- name: Call upload-afs-roots + include_role: + name: upload-afs-roots