Merge "Add general sphinx and reno jobs and role"

playbooks/releasenotes/post.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - role: fetch-sphinx-output
+      sphinx_output_src: "{{ zuul_work_dir|default(zuul.project.src_dir) }}/releasenotes/build/html"

playbooks/releasenotes/pre.yaml

@@ -0,0 +1,12 @@
+- hosts: all
+  roles:
+    - role: bindep
+      bindep_profile: doc
+      bindep_dir: "{{ zuul_work_dir }}"
+    - role: ensure-sphinx
+      doc_building_packages:
+        - sphinx
+        - reno
+      # TODO(jaegerandi): Remove once all repos are fixed.
+      install_package: yes
+    - revoke-sudo

playbooks/releasenotes/run.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - install-if-python
+    - build-releasenotes

playbooks/sphinx/post.yaml

@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - fetch-sphinx-output

playbooks/sphinx/pre.yaml

@@ -0,0 +1,7 @@
+- hosts: all
+  roles:
+    - role: bindep
+      bindep_profile: doc
+      bindep_dir: "{{ zuul_work_dir }}"
+    - ensure-sphinx
+    - revoke-sudo

playbooks/sphinx/run.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - install-if-python
+    - sphinx

roles/build-releasenotes/README.rst

@@ -0,0 +1,13 @@
+Build releasenotes for a project, optionally incorporating translations.
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_work_virtualenv
+   :default: ~/.venv
+
+   Virtualenv location in which to install things.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to build releasenotes in.

roles/build-releasenotes/defaults/main.yaml

@@ -0,0 +1,2 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"

roles/build-releasenotes/tasks/main.yaml

@@ -0,0 +1,129 @@
+# TODO(mordred) Put the translations logic into a sphinx plugin?
+- name: Check if translations exist for release notes
+  stat:
+    path: "{{ zuul_work_dir }}/releasenotes/source/locale"
+    get_checksum: false
+    get_mime: false
+    get_md5: false
+  register: translations
+
+- name: Prepare release note translations
+  shell:
+    chdir: '{{ zuul_work_dir }}'
+    executable: /bin/bash
+    cmd: |
+      set -e
+      set -x
+
+      DOCNAME=releasenotes
+      DIRECTORY=releasenotes
+
+      source {{ zuul_work_virtualenv }}/bin/activate
+
+      # Mapping of language codes to language names
+      declare -A LANG_NAME=(
+          ["de"]="German"
+          ["en_AU"]="English (Australian)"
+          ["en_GB"]="English (United Kingdom)"
+          ["es"]="Spanish"
+          ["fr"]="French"
+          ["id"]="Indonesian"
+          ["it"]="Italian"
+          ["ja"]="Japanese"
+          ["ko_KR"]="Korean (South Korea)"
+          ["pt_BR"]="Portuguese (Brazil)"
+          ["ru"]="Russian"
+          ["tr_TR"]="Turkish (Turkey)"
+          ["zh_CN"]="Chinese (China)"
+      )
+
+      # Check that locale_dirs is really set, otherwise translations
+      # will not work.
+      if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then
+          echo "Translations exist and locale_dirs missing in source/conf.py"
+          exit 1
+      fi
+
+      REFERENCES=`mktemp`
+      trap "rm -f -- '$REFERENCES'" EXIT
+
+      # Extract translations
+      sphinx-build -b gettext \
+          -d ${DIRECTORY}/build/doctrees.gettext \
+          ${DIRECTORY}/source/ \
+          ${DIRECTORY}/source/locale/
+
+      # Add links for translations to index file
+      cat <<EOF >> ${REFERENCES}
+
+      Translated Release Notes
+      ========================
+
+      EOF
+
+      # Check all language translation resources
+      for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do
+          # Skip if it is not a valid language translation resource.
+          if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then
+              continue
+          fi
+          language=$(basename $locale)
+
+          echo "Building $language translation"
+
+          # Prepare all translation resources
+          for pot in ${DIRECTORY}/source/locale/*.pot ; do
+              # Get filename
+              resname=$(basename ${pot} .pot)
+
+              # Merge all translation resources. Note this is done the same
+              # way as done in common_translation_update.sh where we merge
+              # all strings together in a single file.
+              msgmerge --silent -o \
+                  ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \
+                  ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \
+                  ${pot}
+              # Compile all translation resources
+              msgfmt -o \
+                  ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \
+                  ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po
+          done
+
+          # Build translated document
+          sphinx-build -b html -D language=${language} \
+              -d "${DIRECTORY}/build/doctrees.${language}" \
+              ${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language}
+
+          # Reference translated document from index file
+          if [ ${LANG_NAME["${language}"]+_} ] ; then
+              name=${LANG_NAME["${language}"]}
+              name+=" (${language})"
+              echo "* \`$name <${language}/index.html>\`__" >> ${REFERENCES}
+          else
+              echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES}
+          fi
+
+          # Remove newly created files
+          git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po
+          git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo
+          # revert changes to po file
+          git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
+          git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
+      done
+
+      # Now append our references to the index file. We cannot do this
+      # earlier since the sphinx commands will read this file.
+      cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst
+
+      # Remove newly created pot files
+      rm -f ${DIRECTORY}/source/locale/*.pot
+  when: translations.stat.exists == True
+
+- name: Run releasenotes sphinx build
+  shell:
+    executable: /bin/bash
+    chdir: '{{ zuul_work_dir }}'
+    cmd: |
+      {{ zuul_work_virtualenv }}/bin/sphinx-build -a -E -W \
+        -d releasenotes/build/doctrees \
+        -b html releasenotes/source releasenotes/build/html

roles/ensure-sphinx/README.rst

@@ -0,0 +1,27 @@
+Ensure sphinx is installed
+
+Installs sphinx. Also installs any dependencies needed in the first of
+doc/requirements.txt and test-requirements.txt to be found.
+
+All pip installs are done with a provided constraints file, if given.
+
+**Role Variables**
+
+.. zuul:rolevar:: constraints_file
+
+   Optional path to a pip constraints file for installing python libraries.
+
+.. zuul:rolevar:: doc_building_packages
+   :default: ['sphinx']
+
+   List of python packages to install for building docs.
+
+.. zuul:rolevar:: zuul_work_virtualenv
+   :default: ~/.venv
+
+   Virtualenv location in which to install things.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to operate in.

roles/ensure-sphinx/defaults/main.yaml

@@ -0,0 +1,4 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
+doc_building_packages:
+  - sphinx

roles/ensure-sphinx/tasks/main.yaml

@@ -0,0 +1,44 @@
+# TODO(mordred) Make this a list of known binary depends that sphinx needs
+- name: Install gettext package
+  package:
+    name: gettext
+    state: present
+  become: yes
+
+- name: Find Constraints File
+  include_role:
+    name: find-constraints
+
+- name: Install virtualenv and doc requirements files if found
+  shell:
+    executable: /bin/bash
+    chdir: "{{ zuul_work_dir }}"
+    # NOTE(mordred) There is a bug in ansible-lint that mistakenly detects
+    # setting the VENV variable below as an error if it occurs on the fist
+    # line. Work around that by putting a comment as the first line until we
+    # can get a fix upstream.
+    cmd: |
+      # Create virtualenv is it does not already exist
+      VENV={{ zuul_work_virtualenv }}
+      if [ ! -d $VENV ] ; then
+          virtualenv $VENV
+      fi
+      source $VENV/bin/activate
+      # skipping requirements.txt as it gets picked up by installing the
+      # python package itself
+      for f in docs/requirements.txt test-requirements.txt ; do
+          if [ -f $f ] ; then
+              pip install $CONSTRAINTS -r $f
+              break
+          fi
+      done
+  environment:
+    CONSTRAINTS: "{{ upper_constraints|default('') }}"
+
+- name: Install doc building packages
+  pip:
+    name: "{{ item }}"
+    chdir: "{{ zuul_work_dir }}"
+    virtualenv: "{{ zuul_work_virtualenv }}"
+    extra_args: "{{ upper_constraints|default(omit) }}"
+  with_items: "{{ doc_building_packages }}"

roles/find-constraints/README.rst

@@ -0,0 +1,9 @@
+Find a pip constraints file
+
+Sets a variable ``upper_constraints`` which can be passed to a pip invocation.
+
+**Role Variables**
+
+.. zuul:rolevar:: constraints_file
+
+   Optional path to a pip constraints file for installing python libraries.

roles/find-constraints/tasks/main.yaml

@@ -0,0 +1,18 @@
+- name: Check to see if the constraints file exists
+  stat:
+    path: "{{ constraints_file|default('missing') }}"
+    get_checksum: false
+    get_mime: false
+    get_md5: false
+  register: stat_results
+  when: constraints_file is defined
+
+- name: Require defined constraints file to be found
+  fail:
+    msg: constraints_file was defined but was not found on the system
+  when: constraints_file is defined and not stat_results.stat.exists
+
+- name: Record file location
+  set_fact:
+    upper_constraints: "-c {{ constraints_file|realpath }}"
+  when: not stat_results|skipped and stat_results.stat.exists

roles/install-if-python/README.rst

@@ -0,0 +1,30 @@
+Install the contents of a directory if they contain a python project.
+
+Installs into a virtualenv.
+
+**Role Variables**
+
+.. zuul:rolevar:: install_package
+   :default: true
+
+   Flag indicating whether or not the software in the ``zuul_work_dir`` should
+   be installed.
+
+.. zuul:rolevar:: error_on_failure
+
+   Flag that indicates installation errors should result in failure. Failures
+   in installing the target directory are ignored by default.
+
+.. zuul:rolevar:: constraints_file
+
+   Optional path to a pip constraints file to use when installing.
+
+.. zuul:rolevar:: zuul_work_virtualenv
+   :default: ~/.venv
+
+   Virtualenv location in which to install things.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to operate in.

roles/install-if-python/defaults/main.yaml

@@ -0,0 +1,3 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
+install_package: true

roles/install-if-python/tasks/main.yaml

@@ -0,0 +1,59 @@
+# TODO(mordred) rework tox-siblings so it can be used here - probably by
+# making it take a parameter as to what path to python/pip to use.
+
+- name: Find Constraints File
+  include_role:
+    name: find-constraints
+
+- name: Check to see if the project is a python project
+  find:
+    paths: "{{ zuul_work_dir }}"
+    patterns:
+      - setup.cfg
+      - setup.py
+  register: found_python_files
+  when: install_package
+
+# Installing the directory with the constraints flag can hit into problems
+# with conflicting values between constraints and current project. So look
+# for a requirements.txt file so we can install it directly.
+- name: Check to see if the project has a requirements.txt file
+  stat:
+    get_checksum: false
+    get_mime: false
+    get_md5: false
+    path: "{{ zuul_work_dir }}/requirements.txt"
+  register: requirements_file
+
+- name: Install requirements if they exist
+  pip:
+    chdir: "{{ zuul_work_dir }}"
+    virtualenv: "{{ ansible_user_dir }}/.venv"
+    requirements: requirements.txt
+    extra_args: "{{ upper_constraints|default(omit) }}"
+  register: requirements_install
+  when:
+    - install_package
+    - found_python_files.matched
+    - requirements_file.stat.exists
+  failed_when:
+    - error_on_failure is defined
+    - error_on_failure
+    - requirements_install|failed
+
+# Try installing current repo in case it needs to be available for
+# example for version number calculation. Ignore any failures here.
+- name: Install the project if it is a Python project
+  pip:
+    chdir: "{{ zuul_work_dir }}"
+    virtualenv: "{{ ansible_user_dir }}/.venv"
+    name: .
+    extra_args: --no-deps
+  when:
+    - install_package
+    - found_python_files.matched
+  register: install_package_results
+  failed_when:
+    - error_on_failure is defined
+    - error_on_failure
+    - install_package_results|failed

roles/sphinx/README.rst

@@ -0,0 +1,28 @@
+Run sphinx to generate documentation
+
+**Role Variables**
+
+.. zuul:rolevar:: sphinx_source_dir
+   :default: doc/source
+
+   Directory relative to zuul_work_dir that contains the Sphinx sources.
+
+.. zuul:rolevar:: sphinx_build_dir
+   :default: doc/build
+
+   Directory relative to zuul_work_dir where build output will be put.
+
+.. zuul:rolevar:: sphinx_builders
+   :default: ['html']
+
+   Which sphinx builders to run.
+
+.. zuul:rolevar:: zuul_work_virtualenv
+   :default: ~/.venv
+
+   Virtualenv that sphinx is installed in.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to operate in.

roles/sphinx/defaults/main.yaml

@@ -0,0 +1,6 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
+sphinx_source_dir: "doc/source"
+sphinx_build_dir: "doc/build"
+sphinx_builders:
+  - html

roles/sphinx/tasks/main.yaml

@@ -0,0 +1,20 @@
+- name: Run sphinx
+  command:
+    cmd: "{{ zuul_work_virtualenv }}/bin/sphinx-build -b {{ item }} {{ sphinx_source_dir }} {{ sphinx_build_dir }}/{{ item }}"
+    chdir: "{{ zuul_work_dir }}"
+  with_items: "{{ sphinx_builders }}"
+
+- name: Check for whereto
+  stat:
+    path: "{{ zuul_work_virtualenv }}/bin/whereto"
+    get_checksum: false
+    get_mime: false
+    get_md5: false
+  register: whereto
+
+# TODO(mordred) What happens with whereto if sphinx_source_dir is not doc/source?
+- name: Run whereto
+  command:
+    cmd: "{{ whereto.stat.path }} {{ sphinx_source_dir }}/_extra/.htaccess doc/test/redirect-tests.txt"
+    chdir: "{{ zuul_work_dir }}"
+  when: whereto.stat.exists

zuul.yaml

@@ -162,6 +162,67 @@
       at least a ``username`` and ``password`` attribute.
     post-run: playbooks/python/upload-pypi.yaml
 
+- job:
+    name: build-sphinx-docs
+    description: |
+      Build documentation using Sphinx
+
+      Additional requirements can be provided in a project in either the
+      file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first
+      file found in that order will be the one used) Non-python distro
+      requirements can be specified in ``bindep.txt`` using the ``doc`` tag.
+
+      Runs `whereto https://docs.openstack.org/whereto/latest/` after the build
+      if it is installed.
+
+      Responds to these variables:
+
+      .. zuul:jobvar:: constraints_file
+
+         Optional path to a pip constraints file for installing python
+         libraries.
+
+      .. zuul:jobvar:: zuul_work_dir
+         :default: {{ zuul.project.src_dir }}
+
+         Directory to operate in.
+    pre-run: playbooks/sphinx/pre.yaml
+    run: playbooks/sphinx/run.yaml
+    post-run: playbooks/sphinx/post.yaml
+
+- job:
+    name: build-reno-releasenotes
+    description: |
+      Build releasenotes using reno
+
+      Additional requirements can be provided in a project in either the
+      file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first
+      file found in that order will be the one used) Non-python distro
+      requirements can be specified in ``bindep.txt`` using the ``doc`` tag.
+
+      Responds to these variables:
+
+      .. zuul:jobvar:: constraints_file
+
+         Optional path to a pip constraints file for installing python
+         libraries.
+
+      .. zuul:jobvar:: zuul_work_dir
+         :default: {{ zuul.project.src_dir }}
+
+         Directory to operate in.
+    success-url: html/
+    # Release notes always build on master.
+    override-checkout: master
+    pre-run: playbooks/releasenotes/pre.yaml
+    run: playbooks/releasenotes/run.yaml
+    post-run: playbooks/releasenotes/post.yaml
+    files:
+      - ^releasenotes/.*
+      - bindep.txt
+      - doc/requirements.txt
+      - test-requirements.txt
+
 - job:
     name: trigger-readthedocs
     description: Send a trigger to the readthedocs url to tell it to build docs