From b493ca62dd7f39449e799e6f5b84f8b993c03e5f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 17 Nov 2017 09:02:24 -0600 Subject: [PATCH] Add general sphinx and reno jobs and role Sphinx jobs and reno jobs need basically the same thing for dependencies. So make a new role, ensure-sphinx, which is parameterizable enough that it can be used by both reno and sphinx jobs. Make build jobs for both releasenotes and sphinx docs, as both of these things are perfectly valid things to do in both OpenStack and non-OpenStack contexts. We'll add an openstack specific job in openstack-zuul-jobs that uses these as parents but adds the requirements repo and constraints file settings. Some of the pip commands here can be improved once https://github.com/ansible/ansible/pull/33098 lands and is released, which would allow specifying --user and -c as parameters to the pip module. Change-Id: Idd7caf7d88b56d61872906027b4ce7d743572ded Needed-By: I57de14580f39b9e1c11a587b51b44b61b02c84da --- playbooks/releasenotes/post.yaml | 4 + playbooks/releasenotes/pre.yaml | 12 ++ playbooks/releasenotes/run.yaml | 4 + playbooks/sphinx/post.yaml | 3 + playbooks/sphinx/pre.yaml | 7 ++ playbooks/sphinx/run.yaml | 4 + roles/build-releasenotes/README.rst | 13 ++ roles/build-releasenotes/defaults/main.yaml | 2 + roles/build-releasenotes/tasks/main.yaml | 129 ++++++++++++++++++++ roles/ensure-sphinx/README.rst | 27 ++++ roles/ensure-sphinx/defaults/main.yaml | 4 + roles/ensure-sphinx/tasks/main.yaml | 44 +++++++ roles/find-constraints/README.rst | 9 ++ roles/find-constraints/tasks/main.yaml | 18 +++ roles/install-if-python/README.rst | 30 +++++ roles/install-if-python/defaults/main.yaml | 3 + roles/install-if-python/tasks/main.yaml | 59 +++++++++ roles/sphinx/README.rst | 28 +++++ roles/sphinx/defaults/main.yaml | 6 + roles/sphinx/tasks/main.yaml | 20 +++ zuul.yaml | 61 +++++++++ 21 files changed, 487 insertions(+) create mode 100644 playbooks/releasenotes/post.yaml create mode 100644 playbooks/releasenotes/pre.yaml create mode 100644 playbooks/releasenotes/run.yaml create mode 100644 playbooks/sphinx/post.yaml create mode 100644 playbooks/sphinx/pre.yaml create mode 100644 playbooks/sphinx/run.yaml create mode 100644 roles/build-releasenotes/README.rst create mode 100644 roles/build-releasenotes/defaults/main.yaml create mode 100644 roles/build-releasenotes/tasks/main.yaml create mode 100644 roles/ensure-sphinx/README.rst create mode 100644 roles/ensure-sphinx/defaults/main.yaml create mode 100644 roles/ensure-sphinx/tasks/main.yaml create mode 100644 roles/find-constraints/README.rst create mode 100644 roles/find-constraints/tasks/main.yaml create mode 100644 roles/install-if-python/README.rst create mode 100644 roles/install-if-python/defaults/main.yaml create mode 100644 roles/install-if-python/tasks/main.yaml create mode 100644 roles/sphinx/README.rst create mode 100644 roles/sphinx/defaults/main.yaml create mode 100644 roles/sphinx/tasks/main.yaml diff --git a/playbooks/releasenotes/post.yaml b/playbooks/releasenotes/post.yaml new file mode 100644 index 000000000..517617670 --- /dev/null +++ b/playbooks/releasenotes/post.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - role: fetch-sphinx-output + sphinx_output_src: "{{ zuul_work_dir|default(zuul.project.src_dir) }}/releasenotes/build/html" diff --git a/playbooks/releasenotes/pre.yaml b/playbooks/releasenotes/pre.yaml new file mode 100644 index 000000000..3a3a637b3 --- /dev/null +++ b/playbooks/releasenotes/pre.yaml @@ -0,0 +1,12 @@ +- hosts: all + roles: + - role: bindep + bindep_profile: doc + bindep_dir: "{{ zuul_work_dir }}" + - role: ensure-sphinx + doc_building_packages: + - sphinx + - reno + # TODO(jaegerandi): Remove once all repos are fixed. + install_package: yes + - revoke-sudo diff --git a/playbooks/releasenotes/run.yaml b/playbooks/releasenotes/run.yaml new file mode 100644 index 000000000..378cac915 --- /dev/null +++ b/playbooks/releasenotes/run.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - install-if-python + - build-releasenotes diff --git a/playbooks/sphinx/post.yaml b/playbooks/sphinx/post.yaml new file mode 100644 index 000000000..c99e1e609 --- /dev/null +++ b/playbooks/sphinx/post.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - fetch-sphinx-output diff --git a/playbooks/sphinx/pre.yaml b/playbooks/sphinx/pre.yaml new file mode 100644 index 000000000..e3a9ebdb8 --- /dev/null +++ b/playbooks/sphinx/pre.yaml @@ -0,0 +1,7 @@ +- hosts: all + roles: + - role: bindep + bindep_profile: doc + bindep_dir: "{{ zuul_work_dir }}" + - ensure-sphinx + - revoke-sudo diff --git a/playbooks/sphinx/run.yaml b/playbooks/sphinx/run.yaml new file mode 100644 index 000000000..f831207de --- /dev/null +++ b/playbooks/sphinx/run.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - install-if-python + - sphinx diff --git a/roles/build-releasenotes/README.rst b/roles/build-releasenotes/README.rst new file mode 100644 index 000000000..80481ad36 --- /dev/null +++ b/roles/build-releasenotes/README.rst @@ -0,0 +1,13 @@ +Build releasenotes for a project, optionally incorporating translations. + +**Role Variables** + +.. zuul:rolevar:: zuul_work_virtualenv + :default: ~/.venv + + Virtualenv location in which to install things. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to build releasenotes in. diff --git a/roles/build-releasenotes/defaults/main.yaml b/roles/build-releasenotes/defaults/main.yaml new file mode 100644 index 000000000..e95f7bad9 --- /dev/null +++ b/roles/build-releasenotes/defaults/main.yaml @@ -0,0 +1,2 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" diff --git a/roles/build-releasenotes/tasks/main.yaml b/roles/build-releasenotes/tasks/main.yaml new file mode 100644 index 000000000..9396e7c0d --- /dev/null +++ b/roles/build-releasenotes/tasks/main.yaml @@ -0,0 +1,129 @@ +# TODO(mordred) Put the translations logic into a sphinx plugin? +- name: Check if translations exist for release notes + stat: + path: "{{ zuul_work_dir }}/releasenotes/source/locale" + get_checksum: false + get_mime: false + get_md5: false + register: translations + +- name: Prepare release note translations + shell: + chdir: '{{ zuul_work_dir }}' + executable: /bin/bash + cmd: | + set -e + set -x + + DOCNAME=releasenotes + DIRECTORY=releasenotes + + source {{ zuul_work_virtualenv }}/bin/activate + + # Mapping of language codes to language names + declare -A LANG_NAME=( + ["de"]="German" + ["en_AU"]="English (Australian)" + ["en_GB"]="English (United Kingdom)" + ["es"]="Spanish" + ["fr"]="French" + ["id"]="Indonesian" + ["it"]="Italian" + ["ja"]="Japanese" + ["ko_KR"]="Korean (South Korea)" + ["pt_BR"]="Portuguese (Brazil)" + ["ru"]="Russian" + ["tr_TR"]="Turkish (Turkey)" + ["zh_CN"]="Chinese (China)" + ) + + # Check that locale_dirs is really set, otherwise translations + # will not work. + if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then + echo "Translations exist and locale_dirs missing in source/conf.py" + exit 1 + fi + + REFERENCES=`mktemp` + trap "rm -f -- '$REFERENCES'" EXIT + + # Extract translations + sphinx-build -b gettext \ + -d ${DIRECTORY}/build/doctrees.gettext \ + ${DIRECTORY}/source/ \ + ${DIRECTORY}/source/locale/ + + # Add links for translations to index file + cat <> ${REFERENCES} + + Translated Release Notes + ======================== + + EOF + + # Check all language translation resources + for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do + # Skip if it is not a valid language translation resource. + if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then + continue + fi + language=$(basename $locale) + + echo "Building $language translation" + + # Prepare all translation resources + for pot in ${DIRECTORY}/source/locale/*.pot ; do + # Get filename + resname=$(basename ${pot} .pot) + + # Merge all translation resources. Note this is done the same + # way as done in common_translation_update.sh where we merge + # all strings together in a single file. + msgmerge --silent -o \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \ + ${pot} + # Compile all translation resources + msgfmt -o \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po + done + + # Build translated document + sphinx-build -b html -D language=${language} \ + -d "${DIRECTORY}/build/doctrees.${language}" \ + ${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language} + + # Reference translated document from index file + if [ ${LANG_NAME["${language}"]+_} ] ; then + name=${LANG_NAME["${language}"]} + name+=" (${language})" + echo "* \`$name <${language}/index.html>\`__" >> ${REFERENCES} + else + echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES} + fi + + # Remove newly created files + git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po + git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo + # revert changes to po file + git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po + git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po + done + + # Now append our references to the index file. We cannot do this + # earlier since the sphinx commands will read this file. + cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst + + # Remove newly created pot files + rm -f ${DIRECTORY}/source/locale/*.pot + when: translations.stat.exists == True + +- name: Run releasenotes sphinx build + shell: + executable: /bin/bash + chdir: '{{ zuul_work_dir }}' + cmd: | + {{ zuul_work_virtualenv }}/bin/sphinx-build -a -E -W \ + -d releasenotes/build/doctrees \ + -b html releasenotes/source releasenotes/build/html diff --git a/roles/ensure-sphinx/README.rst b/roles/ensure-sphinx/README.rst new file mode 100644 index 000000000..a0922587a --- /dev/null +++ b/roles/ensure-sphinx/README.rst @@ -0,0 +1,27 @@ +Ensure sphinx is installed + +Installs sphinx. Also installs any dependencies needed in the first of +doc/requirements.txt and test-requirements.txt to be found. + +All pip installs are done with a provided constraints file, if given. + +**Role Variables** + +.. zuul:rolevar:: constraints_file + + Optional path to a pip constraints file for installing python libraries. + +.. zuul:rolevar:: doc_building_packages + :default: ['sphinx'] + + List of python packages to install for building docs. + +.. zuul:rolevar:: zuul_work_virtualenv + :default: ~/.venv + + Virtualenv location in which to install things. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. diff --git a/roles/ensure-sphinx/defaults/main.yaml b/roles/ensure-sphinx/defaults/main.yaml new file mode 100644 index 000000000..ddd3976cd --- /dev/null +++ b/roles/ensure-sphinx/defaults/main.yaml @@ -0,0 +1,4 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" +doc_building_packages: + - sphinx diff --git a/roles/ensure-sphinx/tasks/main.yaml b/roles/ensure-sphinx/tasks/main.yaml new file mode 100644 index 000000000..c2c50ef46 --- /dev/null +++ b/roles/ensure-sphinx/tasks/main.yaml @@ -0,0 +1,44 @@ +# TODO(mordred) Make this a list of known binary depends that sphinx needs +- name: Install gettext package + package: + name: gettext + state: present + become: yes + +- name: Find Constraints File + include_role: + name: find-constraints + +- name: Install virtualenv and doc requirements files if found + shell: + executable: /bin/bash + chdir: "{{ zuul_work_dir }}" + # NOTE(mordred) There is a bug in ansible-lint that mistakenly detects + # setting the VENV variable below as an error if it occurs on the fist + # line. Work around that by putting a comment as the first line until we + # can get a fix upstream. + cmd: | + # Create virtualenv is it does not already exist + VENV={{ zuul_work_virtualenv }} + if [ ! -d $VENV ] ; then + virtualenv $VENV + fi + source $VENV/bin/activate + # skipping requirements.txt as it gets picked up by installing the + # python package itself + for f in docs/requirements.txt test-requirements.txt ; do + if [ -f $f ] ; then + pip install $CONSTRAINTS -r $f + break + fi + done + environment: + CONSTRAINTS: "{{ upper_constraints|default('') }}" + +- name: Install doc building packages + pip: + name: "{{ item }}" + chdir: "{{ zuul_work_dir }}" + virtualenv: "{{ zuul_work_virtualenv }}" + extra_args: "{{ upper_constraints|default(omit) }}" + with_items: "{{ doc_building_packages }}" diff --git a/roles/find-constraints/README.rst b/roles/find-constraints/README.rst new file mode 100644 index 000000000..35e8baf7e --- /dev/null +++ b/roles/find-constraints/README.rst @@ -0,0 +1,9 @@ +Find a pip constraints file + +Sets a variable ``upper_constraints`` which can be passed to a pip invocation. + +**Role Variables** + +.. zuul:rolevar:: constraints_file + + Optional path to a pip constraints file for installing python libraries. diff --git a/roles/find-constraints/tasks/main.yaml b/roles/find-constraints/tasks/main.yaml new file mode 100644 index 000000000..4a7e47528 --- /dev/null +++ b/roles/find-constraints/tasks/main.yaml @@ -0,0 +1,18 @@ +- name: Check to see if the constraints file exists + stat: + path: "{{ constraints_file|default('missing') }}" + get_checksum: false + get_mime: false + get_md5: false + register: stat_results + when: constraints_file is defined + +- name: Require defined constraints file to be found + fail: + msg: constraints_file was defined but was not found on the system + when: constraints_file is defined and not stat_results.stat.exists + +- name: Record file location + set_fact: + upper_constraints: "-c {{ constraints_file|realpath }}" + when: not stat_results|skipped and stat_results.stat.exists diff --git a/roles/install-if-python/README.rst b/roles/install-if-python/README.rst new file mode 100644 index 000000000..8b9fc5b96 --- /dev/null +++ b/roles/install-if-python/README.rst @@ -0,0 +1,30 @@ +Install the contents of a directory if they contain a python project. + +Installs into a virtualenv. + +**Role Variables** + +.. zuul:rolevar:: install_package + :default: true + + Flag indicating whether or not the software in the ``zuul_work_dir`` should + be installed. + +.. zuul:rolevar:: error_on_failure + + Flag that indicates installation errors should result in failure. Failures + in installing the target directory are ignored by default. + +.. zuul:rolevar:: constraints_file + + Optional path to a pip constraints file to use when installing. + +.. zuul:rolevar:: zuul_work_virtualenv + :default: ~/.venv + + Virtualenv location in which to install things. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. diff --git a/roles/install-if-python/defaults/main.yaml b/roles/install-if-python/defaults/main.yaml new file mode 100644 index 000000000..4c700228e --- /dev/null +++ b/roles/install-if-python/defaults/main.yaml @@ -0,0 +1,3 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" +install_package: true diff --git a/roles/install-if-python/tasks/main.yaml b/roles/install-if-python/tasks/main.yaml new file mode 100644 index 000000000..91b20e631 --- /dev/null +++ b/roles/install-if-python/tasks/main.yaml @@ -0,0 +1,59 @@ +# TODO(mordred) rework tox-siblings so it can be used here - probably by +# making it take a parameter as to what path to python/pip to use. + +- name: Find Constraints File + include_role: + name: find-constraints + +- name: Check to see if the project is a python project + find: + paths: "{{ zuul_work_dir }}" + patterns: + - setup.cfg + - setup.py + register: found_python_files + when: install_package + +# Installing the directory with the constraints flag can hit into problems +# with conflicting values between constraints and current project. So look +# for a requirements.txt file so we can install it directly. +- name: Check to see if the project has a requirements.txt file + stat: + get_checksum: false + get_mime: false + get_md5: false + path: "{{ zuul_work_dir }}/requirements.txt" + register: requirements_file + +- name: Install requirements if they exist + pip: + chdir: "{{ zuul_work_dir }}" + virtualenv: "{{ ansible_user_dir }}/.venv" + requirements: requirements.txt + extra_args: "{{ upper_constraints|default(omit) }}" + register: requirements_install + when: + - install_package + - found_python_files.matched + - requirements_file.stat.exists + failed_when: + - error_on_failure is defined + - error_on_failure + - requirements_install|failed + +# Try installing current repo in case it needs to be available for +# example for version number calculation. Ignore any failures here. +- name: Install the project if it is a Python project + pip: + chdir: "{{ zuul_work_dir }}" + virtualenv: "{{ ansible_user_dir }}/.venv" + name: . + extra_args: --no-deps + when: + - install_package + - found_python_files.matched + register: install_package_results + failed_when: + - error_on_failure is defined + - error_on_failure + - install_package_results|failed diff --git a/roles/sphinx/README.rst b/roles/sphinx/README.rst new file mode 100644 index 000000000..7bfb34000 --- /dev/null +++ b/roles/sphinx/README.rst @@ -0,0 +1,28 @@ +Run sphinx to generate documentation + +**Role Variables** + +.. zuul:rolevar:: sphinx_source_dir + :default: doc/source + + Directory relative to zuul_work_dir that contains the Sphinx sources. + +.. zuul:rolevar:: sphinx_build_dir + :default: doc/build + + Directory relative to zuul_work_dir where build output will be put. + +.. zuul:rolevar:: sphinx_builders + :default: ['html'] + + Which sphinx builders to run. + +.. zuul:rolevar:: zuul_work_virtualenv + :default: ~/.venv + + Virtualenv that sphinx is installed in. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. diff --git a/roles/sphinx/defaults/main.yaml b/roles/sphinx/defaults/main.yaml new file mode 100644 index 000000000..719eec4fe --- /dev/null +++ b/roles/sphinx/defaults/main.yaml @@ -0,0 +1,6 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" +sphinx_source_dir: "doc/source" +sphinx_build_dir: "doc/build" +sphinx_builders: + - html diff --git a/roles/sphinx/tasks/main.yaml b/roles/sphinx/tasks/main.yaml new file mode 100644 index 000000000..b057231f7 --- /dev/null +++ b/roles/sphinx/tasks/main.yaml @@ -0,0 +1,20 @@ +- name: Run sphinx + command: + cmd: "{{ zuul_work_virtualenv }}/bin/sphinx-build -b {{ item }} {{ sphinx_source_dir }} {{ sphinx_build_dir }}/{{ item }}" + chdir: "{{ zuul_work_dir }}" + with_items: "{{ sphinx_builders }}" + +- name: Check for whereto + stat: + path: "{{ zuul_work_virtualenv }}/bin/whereto" + get_checksum: false + get_mime: false + get_md5: false + register: whereto + +# TODO(mordred) What happens with whereto if sphinx_source_dir is not doc/source? +- name: Run whereto + command: + cmd: "{{ whereto.stat.path }} {{ sphinx_source_dir }}/_extra/.htaccess doc/test/redirect-tests.txt" + chdir: "{{ zuul_work_dir }}" + when: whereto.stat.exists diff --git a/zuul.yaml b/zuul.yaml index d77fb7114..6bfa87033 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -154,6 +154,67 @@ at least a ``username`` and ``password`` attribute. post-run: playbooks/python/upload-pypi.yaml +- job: + name: build-sphinx-docs + description: | + Build documentation using Sphinx + + Additional requirements can be provided in a project in either the + file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first + file found in that order will be the one used) Non-python distro + requirements can be specified in ``bindep.txt`` using the ``doc`` tag. + + Runs `whereto https://docs.openstack.org/whereto/latest/` after the build + if it is installed. + + Responds to these variables: + + .. zuul:jobvar:: constraints_file + + Optional path to a pip constraints file for installing python + libraries. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. + pre-run: playbooks/sphinx/pre.yaml + run: playbooks/sphinx/run.yaml + post-run: playbooks/sphinx/post.yaml + +- job: + name: build-reno-releasenotes + description: | + Build releasenotes using reno + + Additional requirements can be provided in a project in either the + file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first + file found in that order will be the one used) Non-python distro + requirements can be specified in ``bindep.txt`` using the ``doc`` tag. + + Responds to these variables: + + .. zuul:jobvar:: constraints_file + + Optional path to a pip constraints file for installing python + libraries. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. + success-url: html/ + # Release notes always build on master. + override-checkout: master + pre-run: playbooks/releasenotes/pre.yaml + run: playbooks/releasenotes/run.yaml + post-run: playbooks/releasenotes/post.yaml + files: + - ^releasenotes/.* + - bindep.txt + - doc/requirements.txt + - test-requirements.txt + - job: name: trigger-readthedocs description: Send a trigger to the readthedocs url to tell it to build docs