Add general sphinx and reno jobs and role

Sphinx jobs and reno jobs need basically the same thing for
dependencies. So make a new role, ensure-sphinx, which is
parameterizable enough that it can be used by both reno and sphinx jobs.

Make build jobs for both releasenotes and sphinx docs, as both of these
things are perfectly valid things to do in both OpenStack and
non-OpenStack contexts. We'll add an openstack specific job in
openstack-zuul-jobs that uses these as parents but adds the requirements
repo and constraints file settings.

Some of the pip commands here can be improved once
https://github.com/ansible/ansible/pull/33098 lands and is released,
which would allow specifying --user and -c as parameters to the pip
module.

Change-Id: Idd7caf7d88b56d61872906027b4ce7d743572ded
Needed-By: I57de14580f39b9e1c11a587b51b44b61b02c84da
This commit is contained in:
Monty Taylor 2017-11-17 09:02:24 -06:00
parent d0851b0b4a
commit b493ca62dd
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
21 changed files with 487 additions and 0 deletions

View File

@ -0,0 +1,4 @@
- hosts: all
roles:
- role: fetch-sphinx-output
sphinx_output_src: "{{ zuul_work_dir|default(zuul.project.src_dir) }}/releasenotes/build/html"

View File

@ -0,0 +1,12 @@
- hosts: all
roles:
- role: bindep
bindep_profile: doc
bindep_dir: "{{ zuul_work_dir }}"
- role: ensure-sphinx
doc_building_packages:
- sphinx
- reno
# TODO(jaegerandi): Remove once all repos are fixed.
install_package: yes
- revoke-sudo

View File

@ -0,0 +1,4 @@
- hosts: all
roles:
- install-if-python
- build-releasenotes

View File

@ -0,0 +1,3 @@
- hosts: all
roles:
- fetch-sphinx-output

View File

@ -0,0 +1,7 @@
- hosts: all
roles:
- role: bindep
bindep_profile: doc
bindep_dir: "{{ zuul_work_dir }}"
- ensure-sphinx
- revoke-sudo

View File

@ -0,0 +1,4 @@
- hosts: all
roles:
- install-if-python
- sphinx

View File

@ -0,0 +1,13 @@
Build releasenotes for a project, optionally incorporating translations.
**Role Variables**
.. zuul:rolevar:: zuul_work_virtualenv
:default: ~/.venv
Virtualenv location in which to install things.
.. zuul:rolevar:: zuul_work_dir
:default: {{ zuul.project.src_dir }}
Directory to build releasenotes in.

View File

@ -0,0 +1,2 @@
zuul_work_dir: "{{ zuul.project.src_dir }}"
zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"

View File

@ -0,0 +1,129 @@
# TODO(mordred) Put the translations logic into a sphinx plugin?
- name: Check if translations exist for release notes
stat:
path: "{{ zuul_work_dir }}/releasenotes/source/locale"
get_checksum: false
get_mime: false
get_md5: false
register: translations
- name: Prepare release note translations
shell:
chdir: '{{ zuul_work_dir }}'
executable: /bin/bash
cmd: |
set -e
set -x
DOCNAME=releasenotes
DIRECTORY=releasenotes
source {{ zuul_work_virtualenv }}/bin/activate
# Mapping of language codes to language names
declare -A LANG_NAME=(
["de"]="German"
["en_AU"]="English (Australian)"
["en_GB"]="English (United Kingdom)"
["es"]="Spanish"
["fr"]="French"
["id"]="Indonesian"
["it"]="Italian"
["ja"]="Japanese"
["ko_KR"]="Korean (South Korea)"
["pt_BR"]="Portuguese (Brazil)"
["ru"]="Russian"
["tr_TR"]="Turkish (Turkey)"
["zh_CN"]="Chinese (China)"
)
# Check that locale_dirs is really set, otherwise translations
# will not work.
if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then
echo "Translations exist and locale_dirs missing in source/conf.py"
exit 1
fi
REFERENCES=`mktemp`
trap "rm -f -- '$REFERENCES'" EXIT
# Extract translations
sphinx-build -b gettext \
-d ${DIRECTORY}/build/doctrees.gettext \
${DIRECTORY}/source/ \
${DIRECTORY}/source/locale/
# Add links for translations to index file
cat <<EOF >> ${REFERENCES}
Translated Release Notes
========================
EOF
# Check all language translation resources
for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do
# Skip if it is not a valid language translation resource.
if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then
continue
fi
language=$(basename $locale)
echo "Building $language translation"
# Prepare all translation resources
for pot in ${DIRECTORY}/source/locale/*.pot ; do
# Get filename
resname=$(basename ${pot} .pot)
# Merge all translation resources. Note this is done the same
# way as done in common_translation_update.sh where we merge
# all strings together in a single file.
msgmerge --silent -o \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \
${pot}
# Compile all translation resources
msgfmt -o \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po
done
# Build translated document
sphinx-build -b html -D language=${language} \
-d "${DIRECTORY}/build/doctrees.${language}" \
${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language}
# Reference translated document from index file
if [ ${LANG_NAME["${language}"]+_} ] ; then
name=${LANG_NAME["${language}"]}
name+=" (${language})"
echo "* \`$name <${language}/index.html>\`__" >> ${REFERENCES}
else
echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES}
fi
# Remove newly created files
git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po
git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo
# revert changes to po file
git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
done
# Now append our references to the index file. We cannot do this
# earlier since the sphinx commands will read this file.
cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst
# Remove newly created pot files
rm -f ${DIRECTORY}/source/locale/*.pot
when: translations.stat.exists == True
- name: Run releasenotes sphinx build
shell:
executable: /bin/bash
chdir: '{{ zuul_work_dir }}'
cmd: |
{{ zuul_work_virtualenv }}/bin/sphinx-build -a -E -W \
-d releasenotes/build/doctrees \
-b html releasenotes/source releasenotes/build/html

View File

@ -0,0 +1,27 @@
Ensure sphinx is installed
Installs sphinx. Also installs any dependencies needed in the first of
doc/requirements.txt and test-requirements.txt to be found.
All pip installs are done with a provided constraints file, if given.
**Role Variables**
.. zuul:rolevar:: constraints_file
Optional path to a pip constraints file for installing python libraries.
.. zuul:rolevar:: doc_building_packages
:default: ['sphinx']
List of python packages to install for building docs.
.. zuul:rolevar:: zuul_work_virtualenv
:default: ~/.venv
Virtualenv location in which to install things.
.. zuul:rolevar:: zuul_work_dir
:default: {{ zuul.project.src_dir }}
Directory to operate in.

View File

@ -0,0 +1,4 @@
zuul_work_dir: "{{ zuul.project.src_dir }}"
zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
doc_building_packages:
- sphinx

View File

@ -0,0 +1,44 @@
# TODO(mordred) Make this a list of known binary depends that sphinx needs
- name: Install gettext package
package:
name: gettext
state: present
become: yes
- name: Find Constraints File
include_role:
name: find-constraints
- name: Install virtualenv and doc requirements files if found
shell:
executable: /bin/bash
chdir: "{{ zuul_work_dir }}"
# NOTE(mordred) There is a bug in ansible-lint that mistakenly detects
# setting the VENV variable below as an error if it occurs on the fist
# line. Work around that by putting a comment as the first line until we
# can get a fix upstream.
cmd: |
# Create virtualenv is it does not already exist
VENV={{ zuul_work_virtualenv }}
if [ ! -d $VENV ] ; then
virtualenv $VENV
fi
source $VENV/bin/activate
# skipping requirements.txt as it gets picked up by installing the
# python package itself
for f in docs/requirements.txt test-requirements.txt ; do
if [ -f $f ] ; then
pip install $CONSTRAINTS -r $f
break
fi
done
environment:
CONSTRAINTS: "{{ upper_constraints|default('') }}"
- name: Install doc building packages
pip:
name: "{{ item }}"
chdir: "{{ zuul_work_dir }}"
virtualenv: "{{ zuul_work_virtualenv }}"
extra_args: "{{ upper_constraints|default(omit) }}"
with_items: "{{ doc_building_packages }}"

View File

@ -0,0 +1,9 @@
Find a pip constraints file
Sets a variable ``upper_constraints`` which can be passed to a pip invocation.
**Role Variables**
.. zuul:rolevar:: constraints_file
Optional path to a pip constraints file for installing python libraries.

View File

@ -0,0 +1,18 @@
- name: Check to see if the constraints file exists
stat:
path: "{{ constraints_file|default('missing') }}"
get_checksum: false
get_mime: false
get_md5: false
register: stat_results
when: constraints_file is defined
- name: Require defined constraints file to be found
fail:
msg: constraints_file was defined but was not found on the system
when: constraints_file is defined and not stat_results.stat.exists
- name: Record file location
set_fact:
upper_constraints: "-c {{ constraints_file|realpath }}"
when: not stat_results|skipped and stat_results.stat.exists

View File

@ -0,0 +1,30 @@
Install the contents of a directory if they contain a python project.
Installs into a virtualenv.
**Role Variables**
.. zuul:rolevar:: install_package
:default: true
Flag indicating whether or not the software in the ``zuul_work_dir`` should
be installed.
.. zuul:rolevar:: error_on_failure
Flag that indicates installation errors should result in failure. Failures
in installing the target directory are ignored by default.
.. zuul:rolevar:: constraints_file
Optional path to a pip constraints file to use when installing.
.. zuul:rolevar:: zuul_work_virtualenv
:default: ~/.venv
Virtualenv location in which to install things.
.. zuul:rolevar:: zuul_work_dir
:default: {{ zuul.project.src_dir }}
Directory to operate in.

View File

@ -0,0 +1,3 @@
zuul_work_dir: "{{ zuul.project.src_dir }}"
zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
install_package: true

View File

@ -0,0 +1,59 @@
# TODO(mordred) rework tox-siblings so it can be used here - probably by
# making it take a parameter as to what path to python/pip to use.
- name: Find Constraints File
include_role:
name: find-constraints
- name: Check to see if the project is a python project
find:
paths: "{{ zuul_work_dir }}"
patterns:
- setup.cfg
- setup.py
register: found_python_files
when: install_package
# Installing the directory with the constraints flag can hit into problems
# with conflicting values between constraints and current project. So look
# for a requirements.txt file so we can install it directly.
- name: Check to see if the project has a requirements.txt file
stat:
get_checksum: false
get_mime: false
get_md5: false
path: "{{ zuul_work_dir }}/requirements.txt"
register: requirements_file
- name: Install requirements if they exist
pip:
chdir: "{{ zuul_work_dir }}"
virtualenv: "{{ ansible_user_dir }}/.venv"
requirements: requirements.txt
extra_args: "{{ upper_constraints|default(omit) }}"
register: requirements_install
when:
- install_package
- found_python_files.matched
- requirements_file.stat.exists
failed_when:
- error_on_failure is defined
- error_on_failure
- requirements_install|failed
# Try installing current repo in case it needs to be available for
# example for version number calculation. Ignore any failures here.
- name: Install the project if it is a Python project
pip:
chdir: "{{ zuul_work_dir }}"
virtualenv: "{{ ansible_user_dir }}/.venv"
name: .
extra_args: --no-deps
when:
- install_package
- found_python_files.matched
register: install_package_results
failed_when:
- error_on_failure is defined
- error_on_failure
- install_package_results|failed

28
roles/sphinx/README.rst Normal file
View File

@ -0,0 +1,28 @@
Run sphinx to generate documentation
**Role Variables**
.. zuul:rolevar:: sphinx_source_dir
:default: doc/source
Directory relative to zuul_work_dir that contains the Sphinx sources.
.. zuul:rolevar:: sphinx_build_dir
:default: doc/build
Directory relative to zuul_work_dir where build output will be put.
.. zuul:rolevar:: sphinx_builders
:default: ['html']
Which sphinx builders to run.
.. zuul:rolevar:: zuul_work_virtualenv
:default: ~/.venv
Virtualenv that sphinx is installed in.
.. zuul:rolevar:: zuul_work_dir
:default: {{ zuul.project.src_dir }}
Directory to operate in.

View File

@ -0,0 +1,6 @@
zuul_work_dir: "{{ zuul.project.src_dir }}"
zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
sphinx_source_dir: "doc/source"
sphinx_build_dir: "doc/build"
sphinx_builders:
- html

View File

@ -0,0 +1,20 @@
- name: Run sphinx
command:
cmd: "{{ zuul_work_virtualenv }}/bin/sphinx-build -b {{ item }} {{ sphinx_source_dir }} {{ sphinx_build_dir }}/{{ item }}"
chdir: "{{ zuul_work_dir }}"
with_items: "{{ sphinx_builders }}"
- name: Check for whereto
stat:
path: "{{ zuul_work_virtualenv }}/bin/whereto"
get_checksum: false
get_mime: false
get_md5: false
register: whereto
# TODO(mordred) What happens with whereto if sphinx_source_dir is not doc/source?
- name: Run whereto
command:
cmd: "{{ whereto.stat.path }} {{ sphinx_source_dir }}/_extra/.htaccess doc/test/redirect-tests.txt"
chdir: "{{ zuul_work_dir }}"
when: whereto.stat.exists

View File

@ -154,6 +154,67 @@
at least a ``username`` and ``password`` attribute.
post-run: playbooks/python/upload-pypi.yaml
- job:
name: build-sphinx-docs
description: |
Build documentation using Sphinx
Additional requirements can be provided in a project in either the
file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first
file found in that order will be the one used) Non-python distro
requirements can be specified in ``bindep.txt`` using the ``doc`` tag.
Runs `whereto https://docs.openstack.org/whereto/latest/` after the build
if it is installed.
Responds to these variables:
.. zuul:jobvar:: constraints_file
Optional path to a pip constraints file for installing python
libraries.
.. zuul:jobvar:: zuul_work_dir
:default: {{ zuul.project.src_dir }}
Directory to operate in.
pre-run: playbooks/sphinx/pre.yaml
run: playbooks/sphinx/run.yaml
post-run: playbooks/sphinx/post.yaml
- job:
name: build-reno-releasenotes
description: |
Build releasenotes using reno
Additional requirements can be provided in a project in either the
file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first
file found in that order will be the one used) Non-python distro
requirements can be specified in ``bindep.txt`` using the ``doc`` tag.
Responds to these variables:
.. zuul:jobvar:: constraints_file
Optional path to a pip constraints file for installing python
libraries.
.. zuul:jobvar:: zuul_work_dir
:default: {{ zuul.project.src_dir }}
Directory to operate in.
success-url: html/
# Release notes always build on master.
override-checkout: master
pre-run: playbooks/releasenotes/pre.yaml
run: playbooks/releasenotes/run.yaml
post-run: playbooks/releasenotes/post.yaml
files:
- ^releasenotes/.*
- bindep.txt
- doc/requirements.txt
- test-requirements.txt
- job:
name: trigger-readthedocs
description: Send a trigger to the readthedocs url to tell it to build docs