diff --git a/playbooks/releasenotes/post.yaml b/playbooks/releasenotes/post.yaml new file mode 100644 index 000000000..517617670 --- /dev/null +++ b/playbooks/releasenotes/post.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - role: fetch-sphinx-output + sphinx_output_src: "{{ zuul_work_dir|default(zuul.project.src_dir) }}/releasenotes/build/html" diff --git a/playbooks/releasenotes/pre.yaml b/playbooks/releasenotes/pre.yaml new file mode 100644 index 000000000..3a3a637b3 --- /dev/null +++ b/playbooks/releasenotes/pre.yaml @@ -0,0 +1,12 @@ +- hosts: all + roles: + - role: bindep + bindep_profile: doc + bindep_dir: "{{ zuul_work_dir }}" + - role: ensure-sphinx + doc_building_packages: + - sphinx + - reno + # TODO(jaegerandi): Remove once all repos are fixed. + install_package: yes + - revoke-sudo diff --git a/playbooks/releasenotes/run.yaml b/playbooks/releasenotes/run.yaml new file mode 100644 index 000000000..378cac915 --- /dev/null +++ b/playbooks/releasenotes/run.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - install-if-python + - build-releasenotes diff --git a/playbooks/sphinx/post.yaml b/playbooks/sphinx/post.yaml new file mode 100644 index 000000000..c99e1e609 --- /dev/null +++ b/playbooks/sphinx/post.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - fetch-sphinx-output diff --git a/playbooks/sphinx/pre.yaml b/playbooks/sphinx/pre.yaml new file mode 100644 index 000000000..e3a9ebdb8 --- /dev/null +++ b/playbooks/sphinx/pre.yaml @@ -0,0 +1,7 @@ +- hosts: all + roles: + - role: bindep + bindep_profile: doc + bindep_dir: "{{ zuul_work_dir }}" + - ensure-sphinx + - revoke-sudo diff --git a/playbooks/sphinx/run.yaml b/playbooks/sphinx/run.yaml new file mode 100644 index 000000000..f831207de --- /dev/null +++ b/playbooks/sphinx/run.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - install-if-python + - sphinx diff --git a/roles/build-releasenotes/README.rst b/roles/build-releasenotes/README.rst new file mode 100644 index 000000000..80481ad36 --- /dev/null +++ b/roles/build-releasenotes/README.rst @@ -0,0 +1,13 @@ +Build releasenotes for a project, optionally incorporating translations. + +**Role Variables** + +.. zuul:rolevar:: zuul_work_virtualenv + :default: ~/.venv + + Virtualenv location in which to install things. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to build releasenotes in. diff --git a/roles/build-releasenotes/defaults/main.yaml b/roles/build-releasenotes/defaults/main.yaml new file mode 100644 index 000000000..e95f7bad9 --- /dev/null +++ b/roles/build-releasenotes/defaults/main.yaml @@ -0,0 +1,2 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" diff --git a/roles/build-releasenotes/tasks/main.yaml b/roles/build-releasenotes/tasks/main.yaml new file mode 100644 index 000000000..9396e7c0d --- /dev/null +++ b/roles/build-releasenotes/tasks/main.yaml @@ -0,0 +1,129 @@ +# TODO(mordred) Put the translations logic into a sphinx plugin? +- name: Check if translations exist for release notes + stat: + path: "{{ zuul_work_dir }}/releasenotes/source/locale" + get_checksum: false + get_mime: false + get_md5: false + register: translations + +- name: Prepare release note translations + shell: + chdir: '{{ zuul_work_dir }}' + executable: /bin/bash + cmd: | + set -e + set -x + + DOCNAME=releasenotes + DIRECTORY=releasenotes + + source {{ zuul_work_virtualenv }}/bin/activate + + # Mapping of language codes to language names + declare -A LANG_NAME=( + ["de"]="German" + ["en_AU"]="English (Australian)" + ["en_GB"]="English (United Kingdom)" + ["es"]="Spanish" + ["fr"]="French" + ["id"]="Indonesian" + ["it"]="Italian" + ["ja"]="Japanese" + ["ko_KR"]="Korean (South Korea)" + ["pt_BR"]="Portuguese (Brazil)" + ["ru"]="Russian" + ["tr_TR"]="Turkish (Turkey)" + ["zh_CN"]="Chinese (China)" + ) + + # Check that locale_dirs is really set, otherwise translations + # will not work. + if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then + echo "Translations exist and locale_dirs missing in source/conf.py" + exit 1 + fi + + REFERENCES=`mktemp` + trap "rm -f -- '$REFERENCES'" EXIT + + # Extract translations + sphinx-build -b gettext \ + -d ${DIRECTORY}/build/doctrees.gettext \ + ${DIRECTORY}/source/ \ + ${DIRECTORY}/source/locale/ + + # Add links for translations to index file + cat <> ${REFERENCES} + + Translated Release Notes + ======================== + + EOF + + # Check all language translation resources + for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do + # Skip if it is not a valid language translation resource. + if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then + continue + fi + language=$(basename $locale) + + echo "Building $language translation" + + # Prepare all translation resources + for pot in ${DIRECTORY}/source/locale/*.pot ; do + # Get filename + resname=$(basename ${pot} .pot) + + # Merge all translation resources. Note this is done the same + # way as done in common_translation_update.sh where we merge + # all strings together in a single file. + msgmerge --silent -o \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \ + ${pot} + # Compile all translation resources + msgfmt -o \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po + done + + # Build translated document + sphinx-build -b html -D language=${language} \ + -d "${DIRECTORY}/build/doctrees.${language}" \ + ${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language} + + # Reference translated document from index file + if [ ${LANG_NAME["${language}"]+_} ] ; then + name=${LANG_NAME["${language}"]} + name+=" (${language})" + echo "* \`$name <${language}/index.html>\`__" >> ${REFERENCES} + else + echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES} + fi + + # Remove newly created files + git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po + git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo + # revert changes to po file + git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po + git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po + done + + # Now append our references to the index file. We cannot do this + # earlier since the sphinx commands will read this file. + cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst + + # Remove newly created pot files + rm -f ${DIRECTORY}/source/locale/*.pot + when: translations.stat.exists == True + +- name: Run releasenotes sphinx build + shell: + executable: /bin/bash + chdir: '{{ zuul_work_dir }}' + cmd: | + {{ zuul_work_virtualenv }}/bin/sphinx-build -a -E -W \ + -d releasenotes/build/doctrees \ + -b html releasenotes/source releasenotes/build/html diff --git a/roles/ensure-sphinx/README.rst b/roles/ensure-sphinx/README.rst new file mode 100644 index 000000000..a0922587a --- /dev/null +++ b/roles/ensure-sphinx/README.rst @@ -0,0 +1,27 @@ +Ensure sphinx is installed + +Installs sphinx. Also installs any dependencies needed in the first of +doc/requirements.txt and test-requirements.txt to be found. + +All pip installs are done with a provided constraints file, if given. + +**Role Variables** + +.. zuul:rolevar:: constraints_file + + Optional path to a pip constraints file for installing python libraries. + +.. zuul:rolevar:: doc_building_packages + :default: ['sphinx'] + + List of python packages to install for building docs. + +.. zuul:rolevar:: zuul_work_virtualenv + :default: ~/.venv + + Virtualenv location in which to install things. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. diff --git a/roles/ensure-sphinx/defaults/main.yaml b/roles/ensure-sphinx/defaults/main.yaml new file mode 100644 index 000000000..ddd3976cd --- /dev/null +++ b/roles/ensure-sphinx/defaults/main.yaml @@ -0,0 +1,4 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" +doc_building_packages: + - sphinx diff --git a/roles/ensure-sphinx/tasks/main.yaml b/roles/ensure-sphinx/tasks/main.yaml new file mode 100644 index 000000000..c2c50ef46 --- /dev/null +++ b/roles/ensure-sphinx/tasks/main.yaml @@ -0,0 +1,44 @@ +# TODO(mordred) Make this a list of known binary depends that sphinx needs +- name: Install gettext package + package: + name: gettext + state: present + become: yes + +- name: Find Constraints File + include_role: + name: find-constraints + +- name: Install virtualenv and doc requirements files if found + shell: + executable: /bin/bash + chdir: "{{ zuul_work_dir }}" + # NOTE(mordred) There is a bug in ansible-lint that mistakenly detects + # setting the VENV variable below as an error if it occurs on the fist + # line. Work around that by putting a comment as the first line until we + # can get a fix upstream. + cmd: | + # Create virtualenv is it does not already exist + VENV={{ zuul_work_virtualenv }} + if [ ! -d $VENV ] ; then + virtualenv $VENV + fi + source $VENV/bin/activate + # skipping requirements.txt as it gets picked up by installing the + # python package itself + for f in docs/requirements.txt test-requirements.txt ; do + if [ -f $f ] ; then + pip install $CONSTRAINTS -r $f + break + fi + done + environment: + CONSTRAINTS: "{{ upper_constraints|default('') }}" + +- name: Install doc building packages + pip: + name: "{{ item }}" + chdir: "{{ zuul_work_dir }}" + virtualenv: "{{ zuul_work_virtualenv }}" + extra_args: "{{ upper_constraints|default(omit) }}" + with_items: "{{ doc_building_packages }}" diff --git a/roles/find-constraints/README.rst b/roles/find-constraints/README.rst new file mode 100644 index 000000000..35e8baf7e --- /dev/null +++ b/roles/find-constraints/README.rst @@ -0,0 +1,9 @@ +Find a pip constraints file + +Sets a variable ``upper_constraints`` which can be passed to a pip invocation. + +**Role Variables** + +.. zuul:rolevar:: constraints_file + + Optional path to a pip constraints file for installing python libraries. diff --git a/roles/find-constraints/tasks/main.yaml b/roles/find-constraints/tasks/main.yaml new file mode 100644 index 000000000..4a7e47528 --- /dev/null +++ b/roles/find-constraints/tasks/main.yaml @@ -0,0 +1,18 @@ +- name: Check to see if the constraints file exists + stat: + path: "{{ constraints_file|default('missing') }}" + get_checksum: false + get_mime: false + get_md5: false + register: stat_results + when: constraints_file is defined + +- name: Require defined constraints file to be found + fail: + msg: constraints_file was defined but was not found on the system + when: constraints_file is defined and not stat_results.stat.exists + +- name: Record file location + set_fact: + upper_constraints: "-c {{ constraints_file|realpath }}" + when: not stat_results|skipped and stat_results.stat.exists diff --git a/roles/install-if-python/README.rst b/roles/install-if-python/README.rst new file mode 100644 index 000000000..8b9fc5b96 --- /dev/null +++ b/roles/install-if-python/README.rst @@ -0,0 +1,30 @@ +Install the contents of a directory if they contain a python project. + +Installs into a virtualenv. + +**Role Variables** + +.. zuul:rolevar:: install_package + :default: true + + Flag indicating whether or not the software in the ``zuul_work_dir`` should + be installed. + +.. zuul:rolevar:: error_on_failure + + Flag that indicates installation errors should result in failure. Failures + in installing the target directory are ignored by default. + +.. zuul:rolevar:: constraints_file + + Optional path to a pip constraints file to use when installing. + +.. zuul:rolevar:: zuul_work_virtualenv + :default: ~/.venv + + Virtualenv location in which to install things. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. diff --git a/roles/install-if-python/defaults/main.yaml b/roles/install-if-python/defaults/main.yaml new file mode 100644 index 000000000..4c700228e --- /dev/null +++ b/roles/install-if-python/defaults/main.yaml @@ -0,0 +1,3 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" +install_package: true diff --git a/roles/install-if-python/tasks/main.yaml b/roles/install-if-python/tasks/main.yaml new file mode 100644 index 000000000..91b20e631 --- /dev/null +++ b/roles/install-if-python/tasks/main.yaml @@ -0,0 +1,59 @@ +# TODO(mordred) rework tox-siblings so it can be used here - probably by +# making it take a parameter as to what path to python/pip to use. + +- name: Find Constraints File + include_role: + name: find-constraints + +- name: Check to see if the project is a python project + find: + paths: "{{ zuul_work_dir }}" + patterns: + - setup.cfg + - setup.py + register: found_python_files + when: install_package + +# Installing the directory with the constraints flag can hit into problems +# with conflicting values between constraints and current project. So look +# for a requirements.txt file so we can install it directly. +- name: Check to see if the project has a requirements.txt file + stat: + get_checksum: false + get_mime: false + get_md5: false + path: "{{ zuul_work_dir }}/requirements.txt" + register: requirements_file + +- name: Install requirements if they exist + pip: + chdir: "{{ zuul_work_dir }}" + virtualenv: "{{ ansible_user_dir }}/.venv" + requirements: requirements.txt + extra_args: "{{ upper_constraints|default(omit) }}" + register: requirements_install + when: + - install_package + - found_python_files.matched + - requirements_file.stat.exists + failed_when: + - error_on_failure is defined + - error_on_failure + - requirements_install|failed + +# Try installing current repo in case it needs to be available for +# example for version number calculation. Ignore any failures here. +- name: Install the project if it is a Python project + pip: + chdir: "{{ zuul_work_dir }}" + virtualenv: "{{ ansible_user_dir }}/.venv" + name: . + extra_args: --no-deps + when: + - install_package + - found_python_files.matched + register: install_package_results + failed_when: + - error_on_failure is defined + - error_on_failure + - install_package_results|failed diff --git a/roles/sphinx/README.rst b/roles/sphinx/README.rst new file mode 100644 index 000000000..7bfb34000 --- /dev/null +++ b/roles/sphinx/README.rst @@ -0,0 +1,28 @@ +Run sphinx to generate documentation + +**Role Variables** + +.. zuul:rolevar:: sphinx_source_dir + :default: doc/source + + Directory relative to zuul_work_dir that contains the Sphinx sources. + +.. zuul:rolevar:: sphinx_build_dir + :default: doc/build + + Directory relative to zuul_work_dir where build output will be put. + +.. zuul:rolevar:: sphinx_builders + :default: ['html'] + + Which sphinx builders to run. + +.. zuul:rolevar:: zuul_work_virtualenv + :default: ~/.venv + + Virtualenv that sphinx is installed in. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. diff --git a/roles/sphinx/defaults/main.yaml b/roles/sphinx/defaults/main.yaml new file mode 100644 index 000000000..719eec4fe --- /dev/null +++ b/roles/sphinx/defaults/main.yaml @@ -0,0 +1,6 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv" +sphinx_source_dir: "doc/source" +sphinx_build_dir: "doc/build" +sphinx_builders: + - html diff --git a/roles/sphinx/tasks/main.yaml b/roles/sphinx/tasks/main.yaml new file mode 100644 index 000000000..b057231f7 --- /dev/null +++ b/roles/sphinx/tasks/main.yaml @@ -0,0 +1,20 @@ +- name: Run sphinx + command: + cmd: "{{ zuul_work_virtualenv }}/bin/sphinx-build -b {{ item }} {{ sphinx_source_dir }} {{ sphinx_build_dir }}/{{ item }}" + chdir: "{{ zuul_work_dir }}" + with_items: "{{ sphinx_builders }}" + +- name: Check for whereto + stat: + path: "{{ zuul_work_virtualenv }}/bin/whereto" + get_checksum: false + get_mime: false + get_md5: false + register: whereto + +# TODO(mordred) What happens with whereto if sphinx_source_dir is not doc/source? +- name: Run whereto + command: + cmd: "{{ whereto.stat.path }} {{ sphinx_source_dir }}/_extra/.htaccess doc/test/redirect-tests.txt" + chdir: "{{ zuul_work_dir }}" + when: whereto.stat.exists diff --git a/zuul.yaml b/zuul.yaml index d77fb7114..6bfa87033 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -154,6 +154,67 @@ at least a ``username`` and ``password`` attribute. post-run: playbooks/python/upload-pypi.yaml +- job: + name: build-sphinx-docs + description: | + Build documentation using Sphinx + + Additional requirements can be provided in a project in either the + file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first + file found in that order will be the one used) Non-python distro + requirements can be specified in ``bindep.txt`` using the ``doc`` tag. + + Runs `whereto https://docs.openstack.org/whereto/latest/` after the build + if it is installed. + + Responds to these variables: + + .. zuul:jobvar:: constraints_file + + Optional path to a pip constraints file for installing python + libraries. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. + pre-run: playbooks/sphinx/pre.yaml + run: playbooks/sphinx/run.yaml + post-run: playbooks/sphinx/post.yaml + +- job: + name: build-reno-releasenotes + description: | + Build releasenotes using reno + + Additional requirements can be provided in a project in either the + file ``doc/requirements.txt`` or ``test-requirements.txt``. (The first + file found in that order will be the one used) Non-python distro + requirements can be specified in ``bindep.txt`` using the ``doc`` tag. + + Responds to these variables: + + .. zuul:jobvar:: constraints_file + + Optional path to a pip constraints file for installing python + libraries. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to operate in. + success-url: html/ + # Release notes always build on master. + override-checkout: master + pre-run: playbooks/releasenotes/pre.yaml + run: playbooks/releasenotes/run.yaml + post-run: playbooks/releasenotes/post.yaml + files: + - ^releasenotes/.* + - bindep.txt + - doc/requirements.txt + - test-requirements.txt + - job: name: trigger-readthedocs description: Send a trigger to the readthedocs url to tell it to build docs