Add general sphinx and reno jobs and role

Sphinx jobs and reno jobs need basically the same thing for
dependencies. So make a new role, ensure-sphinx, which is
parameterizable enough that it can be used by both reno and sphinx jobs.

Make build jobs for both releasenotes and sphinx docs, as both of these
things are perfectly valid things to do in both OpenStack and
non-OpenStack contexts. We'll add an openstack specific job in
openstack-zuul-jobs that uses these as parents but adds the requirements
repo and constraints file settings.

Some of the pip commands here can be improved once
https://github.com/ansible/ansible/pull/33098 lands and is released,
which would allow specifying --user and -c as parameters to the pip
module.

Change-Id: Idd7caf7d88b56d61872906027b4ce7d743572ded
Needed-By: I57de14580f39b9e1c11a587b51b44b61b02c84da

playbooks/releasenotes/post.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - role: fetch-sphinx-output
+      sphinx_output_src: "{{ zuul_work_dir|default(zuul.project.src_dir) }}/releasenotes/build/html"

playbooks/releasenotes/pre.yaml

@@ -0,0 +1,12 @@
+- hosts: all
+  roles:
+    - role: bindep
+      bindep_profile: doc
+      bindep_dir: "{{ zuul_work_dir }}"
+    - role: ensure-sphinx
+      doc_building_packages:
+        - sphinx
+        - reno
+      # TODO(jaegerandi): Remove once all repos are fixed.
+      install_package: yes
+    - revoke-sudo

playbooks/releasenotes/run.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - install-if-python
+    - build-releasenotes

playbooks/sphinx/post.yaml

@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - fetch-sphinx-output

playbooks/sphinx/pre.yaml

@@ -0,0 +1,7 @@
+- hosts: all
+  roles:
+    - role: bindep
+      bindep_profile: doc
+      bindep_dir: "{{ zuul_work_dir }}"
+    - ensure-sphinx
+    - revoke-sudo

playbooks/sphinx/run.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - install-if-python
+    - sphinx

roles/build-releasenotes/README.rst

@@ -0,0 +1,13 @@
+Build releasenotes for a project, optionally incorporating translations.
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_work_virtualenv
+   :default: ~/.venv
+
+   Virtualenv location in which to install things.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to build releasenotes in.

roles/build-releasenotes/defaults/main.yaml

@@ -0,0 +1,2 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"

roles/build-releasenotes/tasks/main.yaml

@@ -0,0 +1,129 @@
+# TODO(mordred) Put the translations logic into a sphinx plugin?
+- name: Check if translations exist for release notes
+  stat:
+    path: "{{ zuul_work_dir }}/releasenotes/source/locale"
+    get_checksum: false
+    get_mime: false
+    get_md5: false
+  register: translations
+
+- name: Prepare release note translations
+  shell:
+    chdir: '{{ zuul_work_dir }}'
+    executable: /bin/bash
+    cmd: |
+      set -e
+      set -x
+
+      DOCNAME=releasenotes
+      DIRECTORY=releasenotes
+
+      source {{ zuul_work_virtualenv }}/bin/activate
+
+      # Mapping of language codes to language names
+      declare -A LANG_NAME=(
+          ["de"]="German"
+          ["en_AU"]="English (Australian)"
+          ["en_GB"]="English (United Kingdom)"
+          ["es"]="Spanish"
+          ["fr"]="French"
+          ["id"]="Indonesian"
+          ["it"]="Italian"
+          ["ja"]="Japanese"
+          ["ko_KR"]="Korean (South Korea)"
+          ["pt_BR"]="Portuguese (Brazil)"
+          ["ru"]="Russian"
+          ["tr_TR"]="Turkish (Turkey)"
+          ["zh_CN"]="Chinese (China)"
+      )
+
+      # Check that locale_dirs is really set, otherwise translations
+      # will not work.
+      if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then
+          echo "Translations exist and locale_dirs missing in source/conf.py"
+          exit 1
+      fi
+
+      REFERENCES=`mktemp`
+      trap "rm -f -- '$REFERENCES'" EXIT
+
+      # Extract translations
+      sphinx-build -b gettext \
+          -d ${DIRECTORY}/build/doctrees.gettext \
+          ${DIRECTORY}/source/ \
+          ${DIRECTORY}/source/locale/
+
+      # Add links for translations to index file
+      cat <<EOF >> ${REFERENCES}
+
+      Translated Release Notes
+      ========================
+
+      EOF
+
+      # Check all language translation resources
+      for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do
+          # Skip if it is not a valid language translation resource.
+          if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then
+              continue
+          fi
+          language=$(basename $locale)
+
+          echo "Building $language translation"
+
+          # Prepare all translation resources
+          for pot in ${DIRECTORY}/source/locale/*.pot ; do
+              # Get filename
+              resname=$(basename ${pot} .pot)
+
+              # Merge all translation resources. Note this is done the same
+              # way as done in common_translation_update.sh where we merge
+              # all strings together in a single file.
+              msgmerge --silent -o \
+                  ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \
+                  ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \
+                  ${pot}
+              # Compile all translation resources
+              msgfmt -o \
+                  ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \
+                  ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po
+          done
+
+          # Build translated document
+          sphinx-build -b html -D language=${language} \
+              -d "${DIRECTORY}/build/doctrees.${language}" \
+              ${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language}
+
+          # Reference translated document from index file
+          if [ ${LANG_NAME["${language}"]+_} ] ; then
+              name=${LANG_NAME["${language}"]}
+              name+=" (${language})"
+              echo "* \`$name <${language}/index.html>\`__" >> ${REFERENCES}
+          else
+              echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES}
+          fi
+
+          # Remove newly created files
+          git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po
+          git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo
+          # revert changes to po file
+          git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
+          git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
+      done
+
+      # Now append our references to the index file. We cannot do this
+      # earlier since the sphinx commands will read this file.
+      cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst
+
+      # Remove newly created pot files
+      rm -f ${DIRECTORY}/source/locale/*.pot
+  when: translations.stat.exists == True
+
+- name: Run releasenotes sphinx build
+  shell:
+    executable: /bin/bash
+    chdir: '{{ zuul_work_dir }}'
+    cmd: |
+      {{ zuul_work_virtualenv }}/bin/sphinx-build -a -E -W \
+        -d releasenotes/build/doctrees \
+        -b html releasenotes/source releasenotes/build/html

roles/ensure-sphinx/README.rst

@@ -0,0 +1,27 @@
+Ensure sphinx is installed
+
+Installs sphinx. Also installs any dependencies needed in the first of
+doc/requirements.txt and test-requirements.txt to be found.
+
+All pip installs are done with a provided constraints file, if given.
+
+**Role Variables**
+
+.. zuul:rolevar:: constraints_file
+
+   Optional path to a pip constraints file for installing python libraries.
+
+.. zuul:rolevar:: doc_building_packages
+   :default: ['sphinx']
+
+   List of python packages to install for building docs.
+
+.. zuul:rolevar:: zuul_work_virtualenv
+   :default: ~/.venv
+
+   Virtualenv location in which to install things.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to operate in.

roles/ensure-sphinx/defaults/main.yaml

@@ -0,0 +1,4 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
+doc_building_packages:
+  - sphinx

roles/ensure-sphinx/tasks/main.yaml

@@ -0,0 +1,44 @@
+# TODO(mordred) Make this a list of known binary depends that sphinx needs
+- name: Install gettext package
+  package:
+    name: gettext
+    state: present
+  become: yes
+
+- name: Find Constraints File
+  include_role:
+    name: find-constraints
+
+- name: Install virtualenv and doc requirements files if found
+  shell:
+    executable: /bin/bash
+    chdir: "{{ zuul_work_dir }}"
+    # NOTE(mordred) There is a bug in ansible-lint that mistakenly detects
+    # setting the VENV variable below as an error if it occurs on the fist
+    # line. Work around that by putting a comment as the first line until we
+    # can get a fix upstream.
+    cmd: |
+      # Create virtualenv is it does not already exist
+      VENV={{ zuul_work_virtualenv }}
+      if [ ! -d $VENV ] ; then
+          virtualenv $VENV
+      fi
+      source $VENV/bin/activate
+      # skipping requirements.txt as it gets picked up by installing the
+      # python package itself
+      for f in docs/requirements.txt test-requirements.txt ; do
+          if [ -f $f ] ; then
+              pip install $CONSTRAINTS -r $f
+              break
+          fi
+      done
+  environment:
+    CONSTRAINTS: "{{ upper_constraints|default('') }}"
+
+- name: Install doc building packages
+  pip:
+    name: "{{ item }}"
+    chdir: "{{ zuul_work_dir }}"
+    virtualenv: "{{ zuul_work_virtualenv }}"
+    extra_args: "{{ upper_constraints|default(omit) }}"
+  with_items: "{{ doc_building_packages }}"

roles/find-constraints/README.rst

@@ -0,0 +1,9 @@
+Find a pip constraints file
+
+Sets a variable ``upper_constraints`` which can be passed to a pip invocation.
+
+**Role Variables**
+
+.. zuul:rolevar:: constraints_file
+
+   Optional path to a pip constraints file for installing python libraries.

roles/find-constraints/tasks/main.yaml

@@ -0,0 +1,18 @@
+- name: Check to see if the constraints file exists
+  stat:
+    path: "{{ constraints_file|default('missing') }}"
+    get_checksum: false
+    get_mime: false
+    get_md5: false
+  register: stat_results
+  when: constraints_file is defined
+
+- name: Require defined constraints file to be found
+  fail:
+    msg: constraints_file was defined but was not found on the system
+  when: constraints_file is defined and not stat_results.stat.exists
+
+- name: Record file location
+  set_fact:
+    upper_constraints: "-c {{ constraints_file|realpath }}"
+  when: not stat_results|skipped and stat_results.stat.exists

roles/install-if-python/README.rst

@@ -0,0 +1,30 @@
+Install the contents of a directory if they contain a python project.
+
+Installs into a virtualenv.
+
+**Role Variables**
+
+.. zuul:rolevar:: install_package
+   :default: true
+
+   Flag indicating whether or not the software in the ``zuul_work_dir`` should
+   be installed.
+
+.. zuul:rolevar:: error_on_failure
+
+   Flag that indicates installation errors should result in failure. Failures
+   in installing the target directory are ignored by default.
+
+.. zuul:rolevar:: constraints_file
+
+   Optional path to a pip constraints file to use when installing.
+
+.. zuul:rolevar:: zuul_work_virtualenv
+   :default: ~/.venv
+
+   Virtualenv location in which to install things.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to operate in.

roles/install-if-python/defaults/main.yaml

@@ -0,0 +1,3 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
+install_package: true

roles/install-if-python/tasks/main.yaml

@@ -0,0 +1,59 @@
+# TODO(mordred) rework tox-siblings so it can be used here - probably by
+# making it take a parameter as to what path to python/pip to use.
+
+- name: Find Constraints File
+  include_role:
+    name: find-constraints
+
+- name: Check to see if the project is a python project
+  find:
+    paths: "{{ zuul_work_dir }}"
+    patterns:
+      - setup.cfg
+      - setup.py
+  register: found_python_files
+  when: install_package
+
+# Installing the directory with the constraints flag can hit into problems
+# with conflicting values between constraints and current project. So look
+# for a requirements.txt file so we can install it directly.
+- name: Check to see if the project has a requirements.txt file
+  stat:
+    get_checksum: false
+    get_mime: false
+    get_md5: false
+    path: "{{ zuul_work_dir }}/requirements.txt"
+  register: requirements_file
+
+- name: Install requirements if they exist
+  pip:
+    chdir: "{{ zuul_work_dir }}"
+    virtualenv: "{{ ansible_user_dir }}/.venv"
+    requirements: requirements.txt
+    extra_args: "{{ upper_constraints|default(omit) }}"
+  register: requirements_install
+  when:
+    - install_package
+    - found_python_files.matched
+    - requirements_file.stat.exists
+  failed_when:
+    - error_on_failure is defined
+    - error_on_failure
+    - requirements_install|failed
+
+# Try installing current repo in case it needs to be available for
+# example for version number calculation. Ignore any failures here.
+- name: Install the project if it is a Python project
+  pip:
+    chdir: "{{ zuul_work_dir }}"
+    virtualenv: "{{ ansible_user_dir }}/.venv"
+    name: .
+    extra_args: --no-deps
+  when:
+    - install_package
+    - found_python_files.matched
+  register: install_package_results
+  failed_when:
+    - error_on_failure is defined
+    - error_on_failure
+    - install_package_results|failed

roles/sphinx/README.rst

@@ -0,0 +1,28 @@
+Run sphinx to generate documentation
+
+**Role Variables**
+
+.. zuul:rolevar:: sphinx_source_dir
+   :default: doc/source
+
+   Directory relative to zuul_work_dir that contains the Sphinx sources.
+
+.. zuul:rolevar:: sphinx_build_dir
+   :default: doc/build
+
+   Directory relative to zuul_work_dir where build output will be put.
+
+.. zuul:rolevar:: sphinx_builders
+   :default: ['html']
+
+   Which sphinx builders to run.
+
+.. zuul:rolevar:: zuul_work_virtualenv
+   :default: ~/.venv
+
+   Virtualenv that sphinx is installed in.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to operate in.

roles/sphinx/defaults/main.yaml

@@ -0,0 +1,6 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
+sphinx_source_dir: "doc/source"
+sphinx_build_dir: "doc/build"
+sphinx_builders:
+  - html

roles/sphinx/tasks/main.yaml

@@ -0,0 +1,20 @@
+- name: Run sphinx
+  command:
+    cmd: "{{ zuul_work_virtualenv }}/bin/sphinx-build -b {{ item }} {{ sphinx_source_dir }} {{ sphinx_build_dir }}/{{ item }}"
+    chdir: "{{ zuul_work_dir }}"
+  with_items: "{{ sphinx_builders }}"
+
+- name: Check for whereto
+  stat:
+    path: "{{ zuul_work_virtualenv }}/bin/whereto"
+    get_checksum: false
+    get_mime: false
+    get_md5: false
+  register: whereto
+
+# TODO(mordred) What happens with whereto if sphinx_source_dir is not doc/source?
+- name: Run whereto
+  command:
+    cmd: "{{ whereto.stat.path }} {{ sphinx_source_dir }}/_extra/.htaccess doc/test/redirect-tests.txt"
+    chdir: "{{ zuul_work_dir }}"
+  when: whereto.stat.exists

zuul.yaml

@@ -154,6 +154,67 @@
       at least a ``username`` and ``password`` attribute.
     post-run: playbooks/python/upload-pypi.yaml
 
+- job:
+    name: build-sphinx-docs
+    description: |
+      Build documentation using Sphinx
+
+      Additional requirements can be provided in a project in either the
+      file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first
+      file found in that order will be the one used) Non-python distro
+      requirements can be specified in ``bindep.txt`` using the ``doc`` tag.
+
+      Runs `whereto https://docs.openstack.org/whereto/latest/` after the build
+      if it is installed.
+
+      Responds to these variables:
+
+      .. zuul:jobvar:: constraints_file
+
+         Optional path to a pip constraints file for installing python
+         libraries.
+
+      .. zuul:jobvar:: zuul_work_dir
+         :default: {{ zuul.project.src_dir }}
+
+         Directory to operate in.
+    pre-run: playbooks/sphinx/pre.yaml
+    run: playbooks/sphinx/run.yaml
+    post-run: playbooks/sphinx/post.yaml
+
+- job:
+    name: build-reno-releasenotes
+    description: |
+      Build releasenotes using reno
+
+      Additional requirements can be provided in a project in either the
+      file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first
+      file found in that order will be the one used) Non-python distro
+      requirements can be specified in ``bindep.txt`` using the ``doc`` tag.
+
+      Responds to these variables:
+
+      .. zuul:jobvar:: constraints_file
+
+         Optional path to a pip constraints file for installing python
+         libraries.
+
+      .. zuul:jobvar:: zuul_work_dir
+         :default: {{ zuul.project.src_dir }}
+
+         Directory to operate in.
+    success-url: html/
+    # Release notes always build on master.
+    override-checkout: master
+    pre-run: playbooks/releasenotes/pre.yaml
+    run: playbooks/releasenotes/run.yaml
+    post-run: playbooks/releasenotes/post.yaml
+    files:
+      - ^releasenotes/.*
+      - bindep.txt
+      - doc/requirements.txt
+      - test-requirements.txt
+
 - job:
     name: trigger-readthedocs
     description: Send a trigger to the readthedocs url to tell it to build docs