Rework docs jobs to be consumable by tenants

This refactors the docs publication/promote jobs so that there
are base versions which can be consumed by tenants who will supply
their own AFS secrets, as well as versions for use in this tenant
(which consume the base versions in exactly the way another tenant
would).

The documentation policies for the tenant are encoded in the secret,
so each tenant can choose the publication locations and scheme once
and allow all projects within that tenant to use these jobs.

Change-Id: I8f201c0351d4d532ddcbcf1f22f9297dece04fff

playbooks/docs/promote.yaml

@@ -1,10 +1,14 @@
 - hosts: localhost
   tasks:
+    - name: Check execution context
+      when: "zuul.branch is not defined"
+      fail:
+        msg: "This playbook must be run in a branch-based pipeline (e.g., 'promote')."
     - name: Download docs archive
       include_role:
         name: download-artifact
       vars:
-        download_artifact_job: opendev-tox-docs
+        # download_artifact_job: provided by zuul job
         download_artifact_api: "https://zuul.opendev.org/api/tenant/{{ zuul.tenant }}"
         download_artifact_name: docs_archive
         download_artifact_pipeline: gate
@@ -16,35 +20,36 @@
       unarchive:
         src: "{{ zuul.executor.work_root }}/docs-html.tar.bz2"
         dest: "{{ zuul.executor.work_root }}/docs"
-    - name: Write branch marker text
+    - include_role:
-      copy:
+        name: write-root-marker
-        dest: "{{ zuul.executor.work_root }}/docs/.root-marker"
+      vars:
-        content: "Project: {{ zuul.project.name }} Branch: {{ zuul.branch }} Build: {{ zuul.build }} Revision: {{ zuul.ref }}"
+        root_marker_dir: "{{ zuul.executor.work_root }}/docs"
-    - name: Set publication directory to latest
+    - name: Set target directory
       when: "zuul.branch == 'master'"
       set_fact:
-        publication_dir: latest
+        target_dir: "{{ afs.docs_master_path }}"
-    - name: Set publication directory to branch
+    - name: Set target directory
       when: "zuul.branch != 'master'"
       set_fact:
-        publication_dir: "{{ zuul.branch }}"
+        target_dir: "{{ afs.docs_branch_path }}"
     - name: Get an AFS token
       include_role:
         name: create-afs-token
     - name: Create publication directory
       file:
-        path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/"
+        path: "{{ target_dir }}"
         state: directory
     - name: Create redirect htaccess file
+      when: "afs.docs_redirect_path is defined and zuul.branch == 'master'"
       copy:
-        dest: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/.htaccess"
+        dest: "{{ afs.docs_redirect_path }}"
-        content: "Redirect 302 /{{ zuul.project.name }} /{{ zuul.project.name }}/latest"
+        content: "{{ afs.docs_redirect_content }}"
     - name: Upload to AFS
       include_role:
         name: upload-afs
       vars:
         afs_source: "{{ zuul.executor.work_root }}/docs/"
-        afs_target: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/{{ publication_dir }}"
+        afs_target: "{{ target_dir }}"
     - name: Destroy AFS token
       include_role:
         name: destroy-afs-token

playbooks/tox-docs/publish.yaml

@@ -1,26 +1,29 @@
 - hosts: localhost
   tasks:
-    - name: Write branch marker text
+    - name: Check execution context
-      copy:
+      when: "zuul.tag is not defined"
-        dest: "{{ zuul.executor.log_root }}/docs/.root-marker"
+      fail:
-        content: "Project: {{ zuul.project.name }} Branch: {{ zuul.branch }} Build: {{ zuul.build }} Revision: {{ zuul.ref }}"
+        msg: "This playbook must be run in a tag-based pipeline (e.g., 'release')."
-    - name: Set publication directory to tag
+    - include_role:
-      when: "zuul.tag is defined"
+        name: write-root-marker
+      vars:
+        root_marker_dir: "{{ zuul.executor.log_root }}/docs"
+    - name: Set target directory
       set_fact:
-        publication_dir: "{{ zuul.tag }}"
+        target_dir: "{{ afs.docs_tag_path }}"
     - name: Get an AFS token
       include_role:
         name: create-afs-token
     - name: Create publication directory
       file:
-        path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/"
+        path: "{{ target_dir }}"
         state: directory
     - name: Upload to AFS
       include_role:
         name: upload-afs
       vars:
         afs_source: "{{ zuul.executor.log_root }}/docs/"
-        afs_target: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/{{ publication_dir }}"
+        afs_target: "{{ target_dir }}"
     - name: Destroy AFS token
       include_role:
         name: destroy-afs-token

zuul.yaml

@@ -87,6 +87,11 @@
           Jpyp6oXA+W+qefPPqQlxa9EbP3emwVf/HWUdCjlCw+GjDz730P94Xd8ie6KiGq3ywYtFn
           EuJvheVCAYjlq4lwshfNIysNA8WePvAze9T5DpVl1MayR7b1KC+R/7wM6xUCbw=
       service_name: service/opendev-zuul@OPENSTACK.ORG
+      docs_master_path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/latest"
+      docs_branch_path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/{{ zuul.branch }}"
+      docs_tag_path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/{{ zuul.tag }}"
+      docs_redirect_path: "/afs/.openstack.org/project/opendev.org/docs/{{ zuul.project.name }}/.htaccess"
+      docs_redirect_content: "Redirect 302 /{{ zuul.project.name }} /{{ zuul.project.name }}/latest"
 
 - job:
     name: base
@@ -253,9 +258,59 @@
     post-run: playbooks/tox-docs/post.yaml
     success-url: docs/
 
+- job:
+    name: opendev-publish-tox-docs-base
+    # This is not parented to opendev-tox-docs because the post
+    # playbook differs.
+    description: |
+      Publish a ref-based documentation build.
+
+      Use this in the tag or release pipelines to publish a build
+      based on a newly-created tag.
+
+      This is an abstract job intended to be inherited from in an
+      OpenDev tenant and an appropriate secret added.
+
+      .. zuul:jobvar:: afs
+         :type: dict
+
+         This is expected to be a Zuul Secret with these keys:
+
+        .. zuul:jobvar:: keytab
+
+           The AFS keytab for the service principal.
+
+        .. zuul:jobvar:: service_name
+
+           The name of the service princpal.
+
+        .. zuul:jobvar:: docs_master_path
+
+           The full docs publication path to use if the job is run on
+           the master branch.
+
+        .. zuul:jobvar:: docs_branch_path
+
+           The full docs publication path to use if the job is run on
+           any other branch.
+
+        .. zuul:jobvar:: docs_tag_path
+
+           The full docs publication path to use if the job is run on
+           a tag.
+    abstract: True
+    vars:
+      tox_envlist: docs
+      bindep_profile: compile doc
+    pre-run: playbooks/tox-docs/pre.yaml
+    run: playbooks/tox-docs/run.yaml
+    post-run:
+      - playbooks/tox-docs/post.yaml
+      - playbooks/tox-docs/publish.yaml
+
 - job:
     name: opendev-publish-tox-docs
-    parent: opendev-tox-docs
+    parent: opendev-publish-tox-docs-base
     description: |
       Publish a ref-based documentation build.
 
@@ -265,20 +320,81 @@
     secrets:
       - secret: opendev-zuul-docs
         name: afs
+        pass-to-parent: true
 
 - job:
-    name: opendev-promote-docs
+    name: opendev-promote-docs-base
     description: |
       Publish a previously built branch-tip documentation tarball.
 
       Use this in the promote pipeline to publish a branch tip tarball
       built in the gate pipeline.
+
+      This is an abstract job intended to be inherited from in an
+      OpenDev tenant and an appropriate secret added.
+
+      .. zuul:jobvar:: afs
+         :type: dict
+
+         This is expected to be a Zuul Secret with these keys:
+
+        .. zuul:jobvar:: keytab
+
+           The AFS keytab for the service principal.
+
+        .. zuul:jobvar:: service_name
+
+           The name of the service princpal.
+
+        .. zuul:jobvar:: docs_master_path
+
+           The full docs publication path to use if the job is run on
+           the master branch.
+
+        .. zuul:jobvar:: docs_branch_path
+
+           The full docs publication path to use if the job is run on
+           any other branch.
+
+        .. zuul:jobvar:: docs_tag_path
+
+           The full docs publication path to use if the job is run on
+           a tag.
+
+        .. zuul:jobvar:: docs_redirect_path
+
+           If this variable is present, a .htaccess redirect will be
+           created at this path when the job is run on the master
+           branch.  For example, it can be used to redirect "project/"
+           to "project/latest".
+
+        .. zuul:jobvar:: docs_redirect_content
+
+           The contents of the .htaccess file in docs_redirect_path.
+
+      .. zuul:jobvar:: download_artifact_job
+
+         The name of the job which built the docs artifact which this
+         job should download and promote.
+    abstract: True
     run: playbooks/docs/promote.yaml
     nodeset:
       nodes: []
+
+- job:
+    name: opendev-promote-docs
+    parent: opendev-promote-docs-base
+    description: |
+      Publish a previously built branch-tip documentation tarball.
+
+      Use this in the promote pipeline to publish a branch tip tarball
+      built in the gate pipeline.
+    vars:
+      download_artifact_job: opendev-tox-docs
     secrets:
       - secret: opendev-zuul-docs
         name: afs
+        pass-to-parent: true
 
 - project:
     check: