From edab04d5ef4b1e4ff755291909588cb0fe56cbaa Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 1 Apr 2019 11:04:01 -0700 Subject: [PATCH] Rework docs jobs to be consumable by tenants This refactors the docs publication/promote jobs so that there are base versions which can be consumed by tenants who will supply their own AFS secrets, as well as versions for use in this tenant (which consume the base versions in exactly the way another tenant would). The documentation policies for the tenant are encoded in the secret, so each tenant can choose the publication locations and scheme once and allow all projects within that tenant to use these jobs. Change-Id: I8f201c0351d4d532ddcbcf1f22f9297dece04fff --- playbooks/docs/promote.yaml | 31 +++++---- playbooks/tox-docs/publish.yaml | 21 +++--- zuul.yaml | 120 +++++++++++++++++++++++++++++++- 3 files changed, 148 insertions(+), 24 deletions(-) diff --git a/playbooks/docs/promote.yaml b/playbooks/docs/promote.yaml index 9747d97..501b368 100644 --- a/playbooks/docs/promote.yaml +++ b/playbooks/docs/promote.yaml @@ -1,10 +1,14 @@ - hosts: localhost tasks: + - name: Check execution context + when: "zuul.branch is not defined" + fail: + msg: "This playbook must be run in a branch-based pipeline (e.g., 'promote')." - name: Download docs archive include_role: name: download-artifact vars: - download_artifact_job: opendev-tox-docs + # download_artifact_job: provided by zuul job download_artifact_api: "https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}" download_artifact_name: docs_archive download_artifact_pipeline: gate @@ -16,35 +20,36 @@ unarchive: src: "{{ zuul.executor.work_root }}/docs-html.tar.bz2" dest: "{{ zuul.executor.work_root }}/docs" - - name: Write branch marker text - copy: - dest: "{{ zuul.executor.work_root }}/docs/.root-marker" - content: "Project: {{ zuul.project.name }} Branch: {{ zuul.branch }} Build: {{ zuul.build }} Revision: {{ zuul.ref }}" - - name: Set publication directory to latest + - include_role: + name: write-root-marker + vars: + root_marker_dir: "{{ zuul.executor.work_root }}/docs" + - name: Set target directory when: "zuul.branch == 'master'" set_fact: - publication_dir: latest - - name: Set publication directory to branch + target_dir: "{{ afs.docs_master_path }}" + - name: Set target directory when: "zuul.branch != 'master'" set_fact: - publication_dir: "{{ zuul.branch }}" + target_dir: "{{ afs.docs_branch_path }}" - name: Get an AFS token include_role: name: create-afs-token - name: Create publication directory file: - path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/" + path: "{{ target_dir }}" state: directory - name: Create redirect htaccess file + when: "afs.docs_redirect_path is defined and zuul.branch == 'master'" copy: - dest: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/.htaccess" - content: "Redirect 302 /{{ zuul.project.name }} /{{ zuul.project.name }}/latest" + dest: "{{ afs.docs_redirect_path }}" + content: "{{ afs.docs_redirect_content }}" - name: Upload to AFS include_role: name: upload-afs vars: afs_source: "{{ zuul.executor.work_root }}/docs/" - afs_target: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/{{ publication_dir }}" + afs_target: "{{ target_dir }}" - name: Destroy AFS token include_role: name: destroy-afs-token diff --git a/playbooks/tox-docs/publish.yaml b/playbooks/tox-docs/publish.yaml index 9493a50..51e08d7 100644 --- a/playbooks/tox-docs/publish.yaml +++ b/playbooks/tox-docs/publish.yaml @@ -1,26 +1,29 @@ - hosts: localhost tasks: - - name: Write branch marker text - copy: - dest: "{{ zuul.executor.log_root }}/docs/.root-marker" - content: "Project: {{ zuul.project.name }} Branch: {{ zuul.branch }} Build: {{ zuul.build }} Revision: {{ zuul.ref }}" - - name: Set publication directory to tag - when: "zuul.tag is defined" + - name: Check execution context + when: "zuul.tag is not defined" + fail: + msg: "This playbook must be run in a tag-based pipeline (e.g., 'release')." + - include_role: + name: write-root-marker + vars: + root_marker_dir: "{{ zuul.executor.log_root }}/docs" + - name: Set target directory set_fact: - publication_dir: "{{ zuul.tag }}" + target_dir: "{{ afs.docs_tag_path }}" - name: Get an AFS token include_role: name: create-afs-token - name: Create publication directory file: - path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/" + path: "{{ target_dir }}" state: directory - name: Upload to AFS include_role: name: upload-afs vars: afs_source: "{{ zuul.executor.log_root }}/docs/" - afs_target: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/{{ publication_dir }}" + afs_target: "{{ target_dir }}" - name: Destroy AFS token include_role: name: destroy-afs-token diff --git a/zuul.yaml b/zuul.yaml index 0e4d0bc..6a1c2bc 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -87,6 +87,11 @@ Jpyp6oXA+W+qefPPqQlxa9EbP3emwVf/HWUdCjlCw+GjDz730P94Xd8ie6KiGq3ywYtFn EuJvheVCAYjlq4lwshfNIysNA8WePvAze9T5DpVl1MayR7b1KC+R/7wM6xUCbw= service_name: service/opendev-zuul@OPENSTACK.ORG + docs_master_path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/latest" + docs_branch_path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/{{ zuul.branch }}" + docs_tag_path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/{{ zuul.tag }}" + docs_redirect_path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/.htaccess" + docs_redirect_content: "Redirect 302 /{{ zuul.project.name }} /{{ zuul.project.name }}/latest" - job: name: base @@ -253,9 +258,59 @@ post-run: playbooks/tox-docs/post.yaml success-url: docs/ +- job: + name: opendev-publish-tox-docs-base + # This is not parented to opendev-tox-docs because the post + # playbook differs. + description: | + Publish a ref-based documentation build. + + Use this in the tag or release pipelines to publish a build + based on a newly-created tag. + + This is an abstract job intended to be inherited from in an + OpenDev tenant and an appropriate secret added. + + .. zuul:jobvar:: afs + :type: dict + + This is expected to be a Zuul Secret with these keys: + + .. zuul:jobvar:: keytab + + The AFS keytab for the service principal. + + .. zuul:jobvar:: service_name + + The name of the service princpal. + + .. zuul:jobvar:: docs_master_path + + The full docs publication path to use if the job is run on + the master branch. + + .. zuul:jobvar:: docs_branch_path + + The full docs publication path to use if the job is run on + any other branch. + + .. zuul:jobvar:: docs_tag_path + + The full docs publication path to use if the job is run on + a tag. + abstract: True + vars: + tox_envlist: docs + bindep_profile: compile doc + pre-run: playbooks/tox-docs/pre.yaml + run: playbooks/tox-docs/run.yaml + post-run: + - playbooks/tox-docs/post.yaml + - playbooks/tox-docs/publish.yaml + - job: name: opendev-publish-tox-docs - parent: opendev-tox-docs + parent: opendev-publish-tox-docs-base description: | Publish a ref-based documentation build. @@ -265,20 +320,81 @@ secrets: - secret: opendev-zuul-docs name: afs + pass-to-parent: true - job: - name: opendev-promote-docs + name: opendev-promote-docs-base description: | Publish a previously built branch-tip documentation tarball. Use this in the promote pipeline to publish a branch tip tarball built in the gate pipeline. + + This is an abstract job intended to be inherited from in an + OpenDev tenant and an appropriate secret added. + + .. zuul:jobvar:: afs + :type: dict + + This is expected to be a Zuul Secret with these keys: + + .. zuul:jobvar:: keytab + + The AFS keytab for the service principal. + + .. zuul:jobvar:: service_name + + The name of the service princpal. + + .. zuul:jobvar:: docs_master_path + + The full docs publication path to use if the job is run on + the master branch. + + .. zuul:jobvar:: docs_branch_path + + The full docs publication path to use if the job is run on + any other branch. + + .. zuul:jobvar:: docs_tag_path + + The full docs publication path to use if the job is run on + a tag. + + .. zuul:jobvar:: docs_redirect_path + + If this variable is present, a .htaccess redirect will be + created at this path when the job is run on the master + branch. For example, it can be used to redirect "project/" + to "project/latest". + + .. zuul:jobvar:: docs_redirect_content + + The contents of the .htaccess file in docs_redirect_path. + + .. zuul:jobvar:: download_artifact_job + + The name of the job which built the docs artifact which this + job should download and promote. + abstract: True run: playbooks/docs/promote.yaml nodeset: nodes: [] + +- job: + name: opendev-promote-docs + parent: opendev-promote-docs-base + description: | + Publish a previously built branch-tip documentation tarball. + + Use this in the promote pipeline to publish a branch tip tarball + built in the gate pipeline. + vars: + download_artifact_job: opendev-tox-docs secrets: - secret: opendev-zuul-docs name: afs + pass-to-parent: true - project: check: