Translate releasenotes

Add a new script to build release notes including translation,
we will run this script as part of each release notes build.

The new script will build releasenotes and all translations for them.
It checks for available translations and only builds for these.

The translated releasenotes are published in-tree, so for
trove-dashboard this would publish:
/releasenotes/trove-dashboard # Original untranslated content
/releasenotes/trove-dashboard/ja/  # Japanese translation
/releasenotes/trove-dashboard/ko_KR/ # Korean translation

The index file gets automatically enhanced to include
links to the translated releasenotes.

A preview of the generated release notes are at:
http://users.suse.com/~aj/horizon-releasenotes/
http://users.suse.com/~aj/trove-dashboard-releasenotes/

Include the file directly in JJB so that it can be changed anytime.
For details, see:
http://docs.openstack.org/infra/jenkins-job-builder/definition.html#module-jenkins_jobs.local_yaml

This needs a change for jenkins-projects-checks.py: The yaml is now an
extented yaml, use the JJB yaml loader to parse it.

For now, use a separate job gate-{name}-i18n-releasenotes-nv and add it
only to horizon and openstack-manuals so that we can evaluate that this
works as expected. A followup will add this for all jobs and the
publishing as well.

Co-Authored-By: Akihiro Motoki <amotoki@gmail.com>
Change-Id: Ic64d571a91bc1bd292d45c0f6ba1a67bfee75997

jenkins/jobs/include-build-releasenotes.sh

@@ -0,0 +1,117 @@
+#!/bin/bash -xe
+
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+DOCNAME=releasenotes
+DIRECTORY=releasenotes
+
+script_path=/usr/local/jenkins/slave_scripts
+
+# This file always exists in OpenStack CI jobs, check for it so that
+# it can be used manually as well.
+if [ -e "$(pwd)/upper-constraints.txt" ]; then
+    export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
+fi
+
+if [ ! -e ${DIRECTORY}/source/locale/ ]; then
+    echo "No translations found, only building normal release notes"
+    $script_path/run-tox.sh releasenotes
+    exit 0
+fi
+
+# TODO(amotoki): releasenote conf.py files in most projects do not
+# define 'locale_dirs' setting which is required for i18n.
+# Remove this once repositories are changed.
+if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then
+    echo "locale_dirs = ['locale/']" >> $DIRECTORY/source/conf.py
+fi
+
+REFERENCES=`mktemp`
+trap "rm -f -- '$REFERENCES'" EXIT
+
+# Extract translations
+tox -e venv -- sphinx-build -b gettext \
+    -d ${DIRECTORY}/build/doctrees.gettext \
+    ${DIRECTORY}/source/ \
+    ${DIRECTORY}/source/locale/
+
+# Add links for translations to index file
+cat <<EOF >> ${REFERENCES}
+
+Translated Release Notes
+========================
+
+EOF
+
+# Check all language translation resources
+for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do
+    # Skip if it is not a valid language translation resource.
+    if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then
+        continue
+    fi
+    language=$(basename $locale)
+
+    echo "Building $language translation"
+
+    # Prepare all translation resources
+    for pot in ${DIRECTORY}/source/locale/*.pot ; do
+        # Get filename
+        resname=$(basename ${pot} .pot)
+
+        # Merge all translation resources. Note this is done the same
+        # way as done in common_translation_update.sh where we merge
+        # all strings together in a single file.
+        msgmerge --silent -o \
+            ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \
+            ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \
+            ${pot}
+        # Compile all translation resources
+        msgfmt -o \
+            ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \
+            ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po
+    done
+
+    # Build translated document
+    tox -e venv -- sphinx-build -b html -D language=${language} \
+        -d "${DIRECTORY}/build/doctrees.${language}" \
+        ${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language}
+
+    # Reference translated document from index file
+    echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES}
+
+    # Remove newly created files
+    git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po
+    git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo
+    # revert changes to po file
+    git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
+    git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
+done
+
+# Now append our references to the index file. We cannot do this
+# earlier since the sphinx commands will read this file.
+cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst
+
+# Remove newly created pot files
+rm -f ${DIRECTORY}/source/locale/*.pot
+
+# Now build releasenotes with reference to translations
+$script_path/run-tox.sh releasenotes
+
+# Revert any changes to the index file.
+git checkout -- ${DIRECTORY}/source/index.rst
+
+# TODO(amotoki): Revert conf.py now as we might have changed this to
+# enable internationalization.
+# Remove this again later.
+git checkout -- ${DIRECTORY}/source/conf.py

jenkins/jobs/openstack-publish-jobs.yaml

@@ -119,6 +119,22 @@
       - upload-releasenotes-draft
       - console-log
 
+- job-template:
+    name: 'gate-{name}-i18n-releasenotes-nv'
+    node: ubuntu-xenial
+
+    builders:
+      - print-template-name:
+          template-name: "{template-name}"
+      - zuul-git-prep-upper-constraints:
+      - install-distro-packages
+      - revoke-sudo
+      - shell:
+          !include-raw-escape: include-build-releasenotes.sh
+
+    publishers:
+      - upload-releasenotes-draft
+      - console-log
 
 - job-group:
     name: openstack-publish-jobs

jenkins/jobs/projects.yaml

@@ -5414,6 +5414,7 @@
           node:
             - ubuntu-trusty
             - ubuntu-xenial
+      - gate-{name}-i18n-releasenotes-nv
 
 - project:
     name: horizon-cisco-ui
@@ -8620,6 +8621,7 @@
       - gate-{name}-tox-{envlist}-{node}:
           envlist: checksyntax
           node: ubuntu-xenial
+      - gate-{name}-i18n-releasenotes-nv
 
 - project:
     name: openstack-resource-agents-specs

tools/jenkins-projects-checks.py

@@ -14,12 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import io
 import glob
 import sys
-import yaml
-
 import voluptuous as v
 
+# The files uses YAML extensions like !include, therefore use the
+# jenkins-job-builder yaml parser for loading.
+from jenkins_jobs import local_yaml
+
+
 BUILDER = v.Schema({
     v.Required('name'): v.All(str),
     v.Required('builders'): v.All(list),
@@ -63,6 +67,7 @@ PUBLISHER = v.Schema({
     'description': v.All(str)
 })
 
+
 def normalize(s):
     "Normalize string for comparison."
     return s.lower().replace("_", "-")
@@ -114,7 +119,7 @@ def validate_jobs():
     print("=====================")
 
     for job_file in glob.glob('jenkins/jobs/*.yaml'):
-        jobs = yaml.load(open(job_file))
+        jobs = local_yaml.load(io.open(job_file, 'r', encoding='utf-8'))
         for item in jobs:
             if 'builder' in item:
                 schema = BUILDER

tox.ini

@@ -68,6 +68,7 @@ commands =
 deps =
    PyYAML
    voluptuous
+   jenkins-job-builder
 commands =
   {toxinidir}/tools/jenkins-projects-checks.py
 

zuul/layout.yaml

@@ -3319,6 +3319,14 @@ jobs:
       - '^tox.ini'
       - '^.*requirements.txt$'
 
+  - name: ^gate-.*-i18n-releasenotes-nv$
+    success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/releasenotes/build/html/
+    branch: ^(?!stable/(?:juno|kilo)).*$
+    files:
+      - '^releasenotes/.*'
+      - '^tox.ini'
+      - '^.*requirements.txt$'
+
   # Some projects do not have releasenotes until the Mitaka cycle
   - name: ^gate-(keystoneauth|keystonemiddleware|networking-midonet|openstack-manuals|os-client-config|packstack|os-brick|puppet-.*)-releasenotes$
     branch: ^(?!stable/(?:juno|kilo|liberty)).*$
@@ -10020,6 +10028,7 @@ projects:
       - gate-horizon-dsvm-integration-current-ubuntu-xenial
       - gate-horizon-dsvm-integration-deprecated-ubuntu-trusty
       - gate-horizon-dsvm-integration-deprecated-ubuntu-xenial
+      - gate-horizon-i18n-releasenotes-nv
     gate:
       - gate-horizon-tox-py27dj18-ubuntu-trusty
       - gate-horizon-tox-py27dj18-ubuntu-xenial
@@ -12740,6 +12749,7 @@ projects:
       - name: translation-jobs-mitaka
       - name: release-notes-jobs
     check:
+      - gate-openstack-manuals-i18n-releasenotes-nv
       - gate-openstack-manuals-tox-checksyntax-ubuntu-xenial
     gate:
       - gate-openstack-manuals-tox-checksyntax-ubuntu-xenial