diff --git a/roles/prepare-zanata-client/README.rst b/roles/prepare-zanata-client/README.rst new file mode 100644 index 00000000..e4e6d8e3 --- /dev/null +++ b/roles/prepare-zanata-client/README.rst @@ -0,0 +1,37 @@ +Prepare zanata client use + +.. note:: This role is only available for Debian based platforms + currently. + +**Role Variables** + +.. zuul:rolevar:: zanata_api_credentials + + Complex argument which contains the ssh key information. It is + expected that this argument comes from a `Secret` + + .. zuul:rolevar:: server_id + + This is the ID of the zanata server to use + + .. zuul:rolevar:: url + + The url to the zanata server + + .. zuul:rolevar:: username + + The username to use with the zanata server + + .. zuul:rolevar:: key + + The key to login with + +.. zuul:rolevar:: zanata_client_version + :default: 4.3.3 + + The version of zanata client to install + +.. zuul:rolevar:: zanata_client_checksum + :default: 25368516c2c6b94a8ad3397317abf69c723f3ba47a4f0357a31a1e075dd6f810 + + The expected SHA256 checksum of the zanata client diff --git a/roles/prepare-zanata-client/defaults/main.yaml b/roles/prepare-zanata-client/defaults/main.yaml new file mode 100644 index 00000000..500bcda4 --- /dev/null +++ b/roles/prepare-zanata-client/defaults/main.yaml @@ -0,0 +1,3 @@ +--- +zanata_client_version: 4.3.3 +zanata_client_checksum: 25368516c2c6b94a8ad3397317abf69c723f3ba47a4f0357a31a1e075dd6f810 diff --git a/roles/prepare-zanata-client/files/ZanataUtils.py b/roles/prepare-zanata-client/files/ZanataUtils.py new file mode 100755 index 00000000..8803c84a --- /dev/null +++ b/roles/prepare-zanata-client/files/ZanataUtils.py @@ -0,0 +1,203 @@ +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from io import BytesIO +import json +from lxml import etree +import os +import re +import requests +try: + import configparser +except ImportError: + import ConfigParser as configparser +try: + from urllib.parse import urljoin +except ImportError: + from urlparse import urljoin + + +class IniConfig(object): + """Object that stores zanata.ini configuration + + Read zanata.ini and make its values available. + + Attributes: + inifile: The path to the ini file to load values from. + + """ + def __init__(self, inifile): + self.inifile = inifile + self._load_config() + + def _load_config(self): + """Load configuration from the zanata.ini file + + Parses the ini file and stores its data. + + """ + if not os.path.isfile(self.inifile): + raise ValueError('zanata.ini file not found.') + config = configparser.ConfigParser() + try: + config.read(self.inifile) + except configparser.Error: + raise ValueError('zanata.ini could not be parsed, please check ' + 'format.') + for item in config.items('servers'): + item_type = item[0].split('.')[1] + if item_type in ('username', 'key', 'url'): + setattr(self, item_type, item[1]) + + +class ZanataRestService(object): + def __init__(self, zconfig, accept='application/xml', + content_type='application/xml', verify=True): + self.url = zconfig.url + if "charset" not in content_type: + content_type = "%s;charset=utf8" % content_type + self.headers = {'Accept': accept, + 'Content-Type': content_type, + 'X-Auth-User': zconfig.username, + 'X-Auth-Token': zconfig.key} + self.verify = verify + + def _construct_url(self, url_fragment): + return urljoin(self.url, url_fragment) + + def query(self, url_fragment, raise_errors=True): + request_url = self._construct_url(url_fragment) + try: + r = requests.get(request_url, verify=self.verify, + headers=self.headers) + except requests.exceptions.ConnectionError: + raise ValueError('Connection error') + if raise_errors and r.status_code != 200: + raise ValueError('Got status code %s for %s' % + (r.status_code, request_url)) + if raise_errors and not r.content: + raise ValueError('Did not receive any data from %s' % request_url) + return r + + def push(self, url_fragment, data): + request_url = self._construct_url(url_fragment) + try: + return requests.put(request_url, verify=self.verify, + headers=self.headers, data=json.dumps(data)) + except requests.exceptions.ConnectionError: + raise ValueError('Connection error') + + +class ProjectConfig(object): + """Object that stores zanata.xml per-project configuration. + + Write out a zanata.xml file for the project given the supplied values. + + Attributes: + zconfig (IniConfig): zanata.ini values + xmlfile (str): path to zanata.xml to read or write + rules (list): list of two-ples with pattern and rules + """ + def __init__(self, zconfig, xmlfile, rules, verify, **kwargs): + self.rest_service = ZanataRestService(zconfig, verify=verify) + self.xmlfile = xmlfile + self.rules = self._parse_rules(rules) + for key, value in kwargs.items(): + setattr(self, key, value) + self._create_config() + + def _get_tag_prefix(self, root): + """XML utility method + + Get the namespace of the XML file so we can + use it to search for tags. + + """ + return '{%s}' % etree.QName(root).namespace + + def _parse_rules(self, rules): + """Parse a two-ple of pattern, rule. + + Returns a list of dictionaries with 'pattern' and 'rule' keys. + """ + return [{'pattern': rule[0], 'rule': rule[1]} for rule in rules] + + def _create_config(self): + """Create zanata.xml + + Use the supplied parameters to create zanata.xml by downloading + a base version of the file and adding customizations. + + """ + xml = self._fetch_zanata_xml() + self._add_configuration(xml) + self._write_xml(xml) + + def _fetch_zanata_xml(self): + """Get base zanata.xml + + Download a basic version of the configuration for the project + using Zanata's REST API. + + """ + r = self.rest_service.query( + '/rest/projects/p/%s/iterations/i/%s/config' + % (self.project, self.version)) + project_config = r.content + p = etree.XMLParser(remove_blank_text=True) + try: + xml = etree.parse(BytesIO(project_config), p) + except etree.ParseError: + raise ValueError('Error parsing xml output') + return xml + + def _add_configuration(self, xml): + """Insert additional configuration + + Add locale mapping rules to the base zanata.xml retrieved from + the server. + + Args: + xml (etree): zanata.xml file contents + + """ + root = xml.getroot() + s = etree.SubElement(root, 'src-dir') + s.text = self.srcdir + t = etree.SubElement(root, 'trans-dir') + t.text = self.txdir + rules = etree.SubElement(root, 'rules') + for rule in self.rules: + new_rule = etree.SubElement(rules, 'rule') + new_rule.attrib['pattern'] = rule['pattern'] + new_rule.text = rule['rule'] + if self.excludes: + excludes = etree.SubElement(root, 'excludes') + excludes.text = self.excludes + tag_prefix = self._get_tag_prefix(root) + # Work around https://bugzilla.redhat.com/show_bug.cgi?id=1219624 + # by removing port number in URL if it's there + url = root.find('%surl' % tag_prefix) + url.text = re.sub(':443', '', url.text) + + def _write_xml(self, xml): + """Write xml + + Write out xml to zanata.xml. + + """ + try: + xml.write(self.xmlfile, pretty_print=True) + except IOError: + raise ValueError('Error writing zanata.xml.') diff --git a/roles/prepare-zanata-client/files/common_translation_update.sh b/roles/prepare-zanata-client/files/common_translation_update.sh new file mode 100644 index 00000000..8c0a37e4 --- /dev/null +++ b/roles/prepare-zanata-client/files/common_translation_update.sh @@ -0,0 +1,779 @@ +#!/bin/bash -xe +# Common code used by propose_translation_update.sh and +# upstream_translation_update.sh + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $SCRIPTSDIR/common.sh + +# Set start of timestamp for subunit +TRANS_START_TIME=$(date +%s) +SUBUNIT_OUTPUT=testrepository.subunit +# Our images have os-test installed in the following location +TESTR_VENV=/usr/os-testr-env + +# Topic to use for our changes +TOPIC=zanata/translations + +# Used for setup.py babel commands +QUIET="--quiet" + +# Have invalid files been found? +INVALID_PO_FILE=0 + +# ERROR_ABORT signals whether the script aborts with failure, will be +# set to 0 on successful run. +ERROR_ABORT=1 + +# We need a UTF-8 locale, set it properly in case it's not set. +export LANG=en_US.UTF-8 + +trap "finish" EXIT + +# Set up some branch dependent variables +function init_branch { + local branch=$1 + + # The calling environment puts upper-constraints.txt in our + # working directory. + # UPPER_CONSTRAINTS_FILE needs to be exported so that tox can use it + # if needed. + export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt + GIT_BRANCH=$branch +} + + +# Get a module name of a project +function get_modulename { + local project=$1 + local target=$2 + + python $SCRIPTSDIR/get-modulename.py -p $project -t $target +} + +function finish { + + if [[ "$ERROR_ABORT" -eq 1 ]] ; then + $TESTR_VENV/bin/generate-subunit $TRANS_START_TIME $SECONDS \ + 'fail' $JOBNAME >> $SUBUNIT_OUTPUT + + else + $TESTR_VENV/bin/generate-subunit $TRANS_START_TIME $SECONDS \ + 'success' $JOBNAME >> $SUBUNIT_OUTPUT + fi + + gzip -9 $SUBUNIT_OUTPUT + + # Only run this if VENV is setup. + if [ "$VENV" != "" ] ; then + # Delete temporary directories + rm -rf $VENV + VENV="" + fi + if [ "$HORIZON_ROOT" != "" ] ; then + rm -rf $HORIZON_ROOT + HORIZON_ROOT="" + fi +} + +# The ensure-babel and ensure-sphinx roles create a venv in ~/.venv containing +# the needed software. However, it's possible we may want to switch that to +# using pip install --user and ~/.local instead of a venv, so make this +# compatible with either. +function setup_venv { + if [ -d ~/.venv ] ; then + source ~/.venv/bin/activate + else + # Ensure ~/.local/bin is in the path + export PATH=~/.local/bin:$PATH + fi +} + + +# Setup nodejs within the python venv. Match the nodejs version with +# the one used in the nodejs6-npm jobs. +function setup_nodeenv { + + # The ensure-babel and ensure-sphinx roles create a venv in + # ~/.venv containing the needed software. However, it's possible + # we may want to switch that to using pip install --user and + # ~/.local instead of a venv, so make this compatible with either. + + NODE_VENV=~/.local/node_venv + if [ -d ~/.venv ] ; then + pip install nodeenv + nodeenv --node 6.9.4 $NODE_VENV + else + pip install --user nodeenv + ~/.local/bin/nodeenv --node 6.9.4 $NODE_VENV + fi + source $NODE_VENV/bin/activate + +} + +# Setup a project for Zanata. This is used by both Python and Django projects. +# syntax: setup_project +function setup_project { + local project=$1 + local version=$2 + + # Exclude all dot-files, particuarly for things such such as .tox + # and .venv + local exclude='.*/**' + + python $SCRIPTSDIR/create-zanata-xml.py \ + -p $project -v $version --srcdir . --txdir . \ + -r '**/*.pot' '{path}/{locale_with_underscore}/LC_MESSAGES/{filename}.po' \ + -e "$exclude" -f zanata.xml +} + + +# Set global variable DocFolder for manuals projects +function init_manuals { + project=$1 + + DocFolder="doc" + ZanataDocFolder="./doc" + if [ $project = "api-site" -o $project = "security-doc" ] ; then + DocFolder="./" + ZanataDocFolder="." + fi +} + +# Setup project manuals projects (api-site, openstack-manuals, +# security-guide) for Zanata +function setup_manuals { + local project=$1 + local version=${2:-master} + + # Fill in associative array SPECIAL_BOOKS + declare -A SPECIAL_BOOKS + source doc-tools-check-languages.conf + + # Grab all of the rules for the documents we care about + ZANATA_RULES= + + # List of directories to skip. + + # All manuals have a source/common subdirectory that is a symlink + # to doc/common in openstack-manuals. We have to exclude this + # source/common directory everywhere, only doc/common gets + # translated. + EXCLUDE='.*/**,**/source/common/**' + + # Generate pot one by one + for FILE in ${DocFolder}/*; do + # Skip non-directories + if [ ! -d $FILE ]; then + continue + fi + DOCNAME=${FILE#${DocFolder}/} + # Ignore directories that will not get translated + if [[ "$DOCNAME" =~ ^(www|tools|generated|publish-docs)$ ]]; then + continue + fi + IS_RST=0 + if [ ${SPECIAL_BOOKS["${DOCNAME}"]+_} ] ; then + case "${SPECIAL_BOOKS["${DOCNAME}"]}" in + RST) + IS_RST=1 + ;; + skip) + EXCLUDE="$EXCLUDE,${DocFolder}/${DOCNAME}/**" + continue + ;; + esac + fi + if [ ${IS_RST} -eq 1 ] ; then + tox -e generatepot-rst -- ${DOCNAME} + ZANATA_RULES="$ZANATA_RULES -r ${ZanataDocFolder}/${DOCNAME}/source/locale/${DOCNAME}.pot ${DocFolder}/${DOCNAME}/source/locale/{locale_with_underscore}/LC_MESSAGES/${DOCNAME}.po" + else + # Update the .pot file + ./tools/generatepot ${DOCNAME} + if [ -f ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot ]; then + ZANATA_RULES="$ZANATA_RULES -r ${ZanataDocFolder}/${DOCNAME}/locale/${DOCNAME}.pot ${DocFolder}/${DOCNAME}/locale/{locale_with_underscore}.po" + fi + fi + done + + # Project setup and updating POT files for release notes. + if [[ $project == "openstack-manuals" ]] && [[ $version == "master" ]]; then + ZANATA_RULES="$ZANATA_RULES -r ./releasenotes/source/locale/releasenotes.pot releasenotes/source/locale/{locale_with_underscore}/LC_MESSAGES/releasenotes.po" + fi + + python $SCRIPTSDIR/create-zanata-xml.py \ + -p $project -v $version --srcdir . --txdir . \ + $ZANATA_RULES -e "$EXCLUDE" \ + -f zanata.xml +} + +# Setup a training-guides project for Zanata +function setup_training_guides { + local project=training-guides + local version=${1:-master} + + # Update the .pot file + tox -e generatepot-training + + python $SCRIPTSDIR/create-zanata-xml.py \ + -p $project -v $version \ + --srcdir doc/upstream-training/source/locale \ + --txdir doc/upstream-training/source/locale \ + -f zanata.xml +} + +# Setup a i18n project for Zanata +function setup_i18n { + local project=i18n + local version=${1:-master} + + # Update the .pot file + tox -e generatepot + + python $SCRIPTSDIR/create-zanata-xml.py \ + -p $project -v $version \ + --srcdir doc/source/locale \ + --txdir doc/source/locale \ + -f zanata.xml +} + +# Setup a ReactJS project for Zanata +function setup_reactjs_project { + local project=$1 + local version=$2 + + local exclude='node_modules/**' + + setup_nodeenv + + # Extract messages + npm install + npm run build + # Transform them into .pot files + npm run json2pot + + python $SCRIPTSDIR/create-zanata-xml.py \ + -p $project -v $version --srcdir . --txdir . \ + -r '**/*.pot' '{path}/{locale}.po' \ + -e "$exclude" -f zanata.xml +} + +# Setup project so that git review works, sets global variable +# COMMIT_MSG. +function setup_review { + # Note we cannot rely on the default branch in .gitreview being + # correct so we are very explicit here. + local branch=${1:-master} + FULL_PROJECT=$(grep project .gitreview | cut -f2 -d= |sed -e 's/\.git$//') + set +e + read -d '' INITIAL_COMMIT_MSG <&1) + if [ "$trans" = "0 translated messages." ] ; then + rm $pot + # Remove file from git if it's under version control. We previously + # had all pot files under version control, so remove file also + # from git if needed. + git rm --ignore-unmatch $pot + fi +} + +# Run extract_messages for python projects. +function extract_messages_python { + local modulename=$1 + + local pot=${modulename}/locale/${modulename}.pot + + # In case this is an initial run, the locale directory might not + # exist, so create it since extract_messages will fail if it does + # not exist. So, create it if needed. + mkdir -p ${modulename}/locale + + # Update the .pot files + # The "_C" and "_P" prefix are for more-gettext-support blueprint, + # "_C" for message with context, "_P" for plural form message. + pybabel ${QUIET} extract \ + --add-comments Translators: \ + --msgid-bugs-address="https://bugs.launchpad.net/openstack-i18n/" \ + --project=${PROJECT} --version=${VERSION} \ + -k "_C:1c,2" -k "_P:1,2" \ + -o ${pot} ${modulename} + check_empty_pot ${pot} +} + +# Django projects need horizon installed for extraction, install it in +# our venv. The function setup_venv needs to be called first. +function install_horizon { + + # TODO(mordred) Replace this with something else that uses the horizon + # repo on disk + HORIZON_ROOT=$(mktemp -d) + + # Checkout same branch of horizon as the project - including + # same constraints. + git clone --depth=1 --branch $GIT_BRANCH \ + https://git.openstack.org/openstack/horizon.git $HORIZON_ROOT/horizon + (cd ${HORIZON_ROOT}/horizon && pip install -c $UPPER_CONSTRAINTS_FILE .) + rm -rf HORIZON_ROOT + HORIZON_ROOT="" +} + + +# Extract messages for a django project, we need to update django.pot +# and djangojs.pot. +function extract_messages_django { + local modulename=$1 + local pot + + KEYWORDS="-k gettext_noop -k gettext_lazy -k ngettext_lazy:1,2" + KEYWORDS+=" -k ugettext_noop -k ugettext_lazy -k ungettext_lazy:1,2" + KEYWORDS+=" -k npgettext:1c,2,3 -k pgettext_lazy:1c,2 -k npgettext_lazy:1c,2,3" + + for DOMAIN in djangojs django ; do + if [ -f babel-${DOMAIN}.cfg ]; then + mkdir -p ${modulename}/locale + pot=${modulename}/locale/${DOMAIN}.pot + touch ${pot} + pybabel ${QUIET} extract -F babel-${DOMAIN}.cfg \ + --add-comments Translators: \ + --msgid-bugs-address="https://bugs.launchpad.net/openstack-i18n/" \ + --project=${PROJECT} --version=${VERSION} \ + $KEYWORDS \ + -o ${pot} ${modulename} + check_empty_pot ${pot} + fi + done +} + +# Extract doc messages +function extract_messages_doc { + # Temporary build folder for gettext + mkdir -p doc/build/gettext + + # Extract messages + sphinx-build -b gettext doc/source \ + doc/build/gettext/ + # Manipulates pot translation sources if needed + if [[ -f tools/doc-pot-filter.sh ]]; then + tools/doc-pot-filter.sh + fi + + # New translation target projects may not have locale folder + mkdir -p doc/source/locale + + # Sphinx builds a pot file for each directory and for each file + # in the top-level directory. + # We keep the directory files and concatenate all top-level files. + local has_other=0 + for f in doc/build/gettext/*.pot; do + local fn=$(basename $f .pot) + # If a pot file corresponds to a directory, we use the pot file as-is. + if [ -d doc/source/$fn ]; then + # Remove UUIDs, those are not necessary and change too often + msgcat --use-first --sort-by-file $f | \ + awk '$0 !~ /^\# [a-z0-9]+$/' > doc/source/locale/doc-$fn.pot + rm $f + else + has_other=1 + fi + done + + # We concatenate remaining into a single pot file so that + # "git add ${DIRECTORY}/source/locale" will only add a + # single pot file for all top-level files. + if [ "$has_other" = "1" ]; then + # Remove UUIDs, those are not necessary and change too often + msgcat --use-first --sort-by-file doc/build/gettext/*.pot | \ + awk '$0 !~ /^\# [a-z0-9]+$/' > doc/source/locale/doc.pot + fi + + rm -rf doc/build/gettext/ +} + +# Extract releasenotes messages +function extract_messages_releasenotes { + local keep_workdir=$1 + + # Extract messages + $HOME/.venv/bin/sphinx-build -b gettext -d releasenotes/build/doctrees \ + releasenotes/source releasenotes/work + rm -rf releasenotes/build + # Concatenate messages into one POT file + mkdir -p releasenotes/source/locale/ + msgcat --sort-by-file releasenotes/work/*.pot \ + > releasenotes/source/locale/releasenotes.pot + if [ ! -n "$keep_workdir" ]; then + rm -rf releasenotes/work + fi +} + +# Check releasenote translation progress per language. +# It checks the progress per release. Add the release note translation +# if at least one release is well translated (>= 75%). +# Keep the release note translation in the git repository +# if at least one release is translated >= 40%. +# Otherwise (< 40%) the translation are removed. +# +# NOTE: this function assume POT files in releasenotes/work +# extracted by extract_messages_releasenotes(). +# The workdir should be clean up by the caller. +function check_releasenotes_per_language { + local lang_po=$1 + + # The expected PO location is + # releasenotes/source/locale//LC_MESSAGES/releasenotes.po. + # Extract language name from 4th component. + local lang + lang=$(echo $lang_po | cut -d / -f 4) + + local release_pot + local release_name + local workdir=releasenotes/work + + local has_high_thresh=0 + local has_low_thresh=0 + + mkdir -p $workdir/$lang + for release_pot in $(find $workdir -name '*.pot'); do + release_name=$(basename $release_pot .pot) + # The index file usually contains small number of words, + # so we skip to check it. + if [ $release_name = "index" ]; then + continue + fi + msgmerge --quiet -o $workdir/$lang/$release_name.po $lang_po $release_pot + check_po_file $workdir/$lang/$release_name.po + if [ $RATIO -ge 75 ]; then + has_high_thresh=1 + has_low_thresh=1 + fi + if [ $RATIO -ge 40 ]; then + has_low_thresh=1 + fi + done + + if ! git ls-files | grep -xq $lang_po; then + if [ $has_high_thresh -eq 0 ]; then + rm -f $lang_po + fi + else + if [ $has_low_thresh -eq 0 ]; then + git rm -f --ignore-unmatch $lang_po + fi + fi +} + +# Filter out files that we do not want to commit. +# Sets global variable INVALID_PO_FILE to 1 if any invalid files are +# found. +function filter_commits { + local ret + + # Don't add new empty files. + for f in $(git diff --cached --name-only --diff-filter=A); do + case "$f" in + *.po) + # Files should have at least one non-empty msgid string. + if ! grep -q 'msgid "[^"]' "$f" ; then + git reset -q "$f" + rm "$f" + fi + ;; + *.json) + # JSON files fail msgid test. Ignore the locale key and confirm + # there are string keys in the messages dictionary itself. + if ! grep -q '"[^"].*":\s*"' "$f" ; then + git reset -q "$f" + rm "$f" + fi + ;; + *) + # Anything else is not a translation file, remove it. + git reset -q "$f" + rm "$f" + ;; + esac + done + + # Don't send files where the only things which have changed are + # the creation date, the version number, the revision date, + # name of last translator, comment lines, or diff file information. + REAL_CHANGE=0 + # Always remove obsolete log level translations. + for f in $(git diff --cached --name-only --diff-filter=D); do + if [[ $f =~ -log-(critical|error|info|warning).po$ ]] ; then + REAL_CHANGE=1 + fi + done + # Don't iterate over deleted files + for f in $(git diff --cached --name-only --diff-filter=AM); do + # It's ok if the grep fails + set +e + REGEX="(POT-Creation-Date|Project-Id-Version|PO-Revision-Date|Last-Translator|X-Generator|Generated-By)" + changed=$(git diff --cached "$f" \ + | egrep -v "$REGEX" \ + | egrep -c "^([-+][^-+#])") + added=$(git diff --cached "$f" \ + | egrep -v "$REGEX" \ + | egrep -c "^([+][^+#])") + set -e + # Check that imported po files are valid + if [[ $f =~ .po$ ]] ; then + set +e + msgfmt --check-format -o /dev/null $f + ret=$? + set -e + if [ $ret -ne 0 ] ; then + # Set change to zero so that next expression reverts + # change of this file. + changed=0 + echo "ERROR: File $f is an invalid po file." + echo "ERROR: The file has not been imported and needs fixing!" + INVALID_PO_FILE=1 + fi + fi + if [ $changed -eq 0 ]; then + git reset -q "$f" + git checkout -- "$f" + # We will take this import if at least one change adds new content, + # thus adding a new translation. + # If only lines are removed, we do not need to generate an import. + elif [ $added -gt 0 ] ; then + REAL_CHANGE=1 + fi + done + + # If no file has any real change, revert all changes. + if [ $REAL_CHANGE -eq 0 ] ; then + # New files need to be handled differently + for f in $(git diff --cached --name-only --diff-filter=A) ; do + git reset -q -- "$f" + rm "$f" + done + for f in $(git diff --cached --name-only) ; do + git reset -q -- "$f" + git checkout -- "$f" + done + fi +} + +# Check the amount of translation done for a .po file, sets global variable +# RATIO. +function check_po_file { + local file=$1 + local dropped_ratio=$2 + + trans=$(msgfmt --statistics -o /dev/null "$file" 2>&1) + check="^0 translated messages" + if [[ $trans =~ $check ]] ; then + RATIO=0 + else + if [[ $trans =~ " translated message" ]] ; then + trans_no=$(echo $trans|sed -e 's/ translated message.*$//') + else + trans_no=0 + fi + if [[ $trans =~ " untranslated message" ]] ; then + untrans_no=$(echo $trans|sed -e 's/^.* \([0-9]*\) untranslated message.*/\1/') + else + untrans_no=0 + fi + total=$(($trans_no+$untrans_no)) + RATIO=$((100*$trans_no/$total)) + fi +} + +# Remove obsolete files. We might have added them in the past but +# would not add them today, so let's eventually remove them. +function cleanup_po_files { + local modulename=$1 + + for i in $(find $modulename -name *.po) ; do + check_po_file "$i" + if [ $RATIO -lt 40 ]; then + git rm -f --ignore-unmatch $i + fi + done +} + +# Remove obsolete log lovel files. We have added them in the past but +# do not translate them anymore, so let's eventually remove them. +function cleanup_log_files { + local modulename=$1 + local levels="info warning error critical" + + for i in $(find $modulename -name *.po) ; do + # We do not store the log level files anymore, remove them + # from git. + local bi + bi=$(basename $i) + + for level in $levels ; do + if [[ "$bi" == "$modulename-log-$level.po" ]] ; then + git rm -f --ignore-unmatch $i + fi + done + done +} + + +# Remove all pot files, we publish them to +# http://tarballs.openstack.org/translation-source/{name}/VERSION , +# let's not store them in git at all. +# Previously, we had those files in git, remove them now if there +# are still there. +function cleanup_pot_files { + local modulename=$1 + + for i in $(find $modulename -name *.pot) ; do + # Remove file; both local and from git if needed. + rm $i + git rm -f --ignore-unmatch $i + done +} + +# Reduce size of po files. This reduces the amount of content imported +# and makes for fewer imports. +# This does not touch the pot files. This way we can reconstruct the po files +# using "msgmerge POTFILE POFILE -o COMPLETEPOFILE". +function compress_po_files { + local directory=$1 + + for i in $(find $directory -name *.po) ; do + msgattrib --translated --no-location --sort-output "$i" \ + --output="${i}.tmp" + mv "${i}.tmp" "$i" + done +} + +function pull_from_zanata { + + local project=$1 + + # Since Zanata does not currently have an option to not download new + # files, we download everything, and then remove new files that are not + # translated enough. + zanata-cli -B -e pull + + # We skip directories starting with '.' because they never contain + # translations for the project (in particular, '.tox'). Likewise + # 'node_modules' only contains dependencies and should be ignored. + for i in $(find . -name '*.po' ! -path './.*' ! -path './node_modules/*' -prune | cut -b3-); do + # We check release note translation percentage per release. + # To check this we need to extract messages per RST file. + # Let's defer checking it to propose_releasenotes. + local basefn= + if [ "$(basename $i)" = "releasenotes.po" ]; then + continue + fi + check_po_file "$i" + # We want new files to be >75% translated. The + # common documents in openstack-manuals have that relaxed to + # >40%. + percentage=75 + if [ $project = "openstack-manuals" ]; then + case "$i" in + *common*) + percentage=40 + ;; + esac + fi + if [ $RATIO -lt $percentage ]; then + # This means the file is below the ratio, but we only want + # to delete it, if it is a new file. Files known to git + # that drop below 40% will be cleaned up by + # cleanup_po_files. + if ! git ls-files | grep -xq "$i"; then + rm -f "$i" + fi + fi + done +} + +# Copy all pot files in modulename directory to temporary path for +# publishing. This uses the exact same path. +function copy_pot { + local all_modules=$1 + local target=.translation-source/$ZANATA_VERSION/ + + for m in $all_modules ; do + for f in `find $m -name "*.pot" ` ; do + local fd + fd=$(dirname $f) + mkdir -p $target/$fd + cp $f $target/$f + done + done +} diff --git a/roles/prepare-zanata-client/files/create-zanata-xml.py b/roles/prepare-zanata-client/files/create-zanata-xml.py new file mode 100755 index 00000000..b58d8c2e --- /dev/null +++ b/roles/prepare-zanata-client/files/create-zanata-xml.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os +import sys +from ZanataUtils import IniConfig, ProjectConfig + + +def get_args(): + parser = argparse.ArgumentParser(description='Generate a zanata.xml ' + 'file for this project so we can ' + 'process translations') + parser.add_argument('-p', '--project') + parser.add_argument('-v', '--version') + parser.add_argument('-s', '--srcdir') + parser.add_argument('-d', '--txdir') + parser.add_argument('-e', '--excludes') + parser.add_argument('-r', '--rule', nargs=2, metavar=('PATTERN', 'RULE'), + action='append', + help='Append a rule, used by the Zanata client to ' + 'match .pot files to translations. Can be specified ' + 'multiple times, and if no rules are specified a ' + 'default will be used.') + parser.add_argument('--no-verify', action='store_false', dest='verify', + help='Do not perform HTTPS certificate verification') + parser.add_argument('-f', '--file', required=True) + return parser.parse_args() + + +def main(): + args = get_args() + default_rule = ('**/*.pot', + '{locale_with_underscore}/LC_MESSAGES/{filename}.po') + rules = args.rule or [default_rule] + try: + zc = IniConfig(os.path.expanduser('~/.config/zanata.ini')) + ProjectConfig(zc, args.file, rules, args.verify, project=args.project, + version=args.version, + srcdir=args.srcdir, txdir=args.txdir, + excludes=args.excludes) + except ValueError as e: + sys.exit(e) + + +if __name__ == '__main__': + main() diff --git a/roles/prepare-zanata-client/files/get-modulename.py b/roles/prepare-zanata-client/files/get-modulename.py new file mode 100755 index 00000000..e56fb388 --- /dev/null +++ b/roles/prepare-zanata-client/files/get-modulename.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import argparse +import ConfigParser as configparser +import sys + + +DJANGO_PROJECT_SUFFIXES = ( + '-dashboard', + 'horizon', # to match horizon and *-horizon + '-ui', + 'django_openstack_auth', +) + + +def get_args(): + parser = argparse.ArgumentParser( + description='Find module names in a repository.') + parser.add_argument('-p', '--project', required=True) + parser.add_argument('-t', '--target', + choices=['python', 'django'], + default='python', + help='Type of modules to search (default: python).') + parser.add_argument('-f', '--file', default='setup.cfg', + help='Path of setup.cfg file.') + return parser.parse_args() + + +def split_multiline(value): + value = [element for element in + (line.strip() for line in value.split('\n')) + if element] + return value + + +def read_config(path): + parser = configparser.SafeConfigParser() + parser.read(path) + + config = {} + for section in parser.sections(): + config[section] = dict(parser.items(section)) + return config + + +def get_option(config, section, option, multiline=False): + if section not in config: + return + value = config[section].get(option) + if not value: + return + if multiline: + value = split_multiline(value) + return value + + +def get_translate_options(config, target): + translate_options = {} + for key, value in config['openstack_translations'].items(): + values = split_multiline(value) + if values: + translate_options[key] = values + return translate_options.get('%s_modules' % target, []) + + +def get_valid_modules(config, project, target): + is_django = any(project.endswith(suffix) + for suffix in DJANGO_PROJECT_SUFFIXES) + if is_django != (target == 'django'): + return [] + + modules = get_option(config, 'files', 'packages', multiline=True) + # If setup.cfg does not contain [files] packages entry, + # let's assume the project name as a module name. + if not modules: + print('[files] packages entry not found in setup.cfg. ' + 'Use project name "%s" as a module name.' % project, + file=sys.stderr) + modules = [project] + return modules + + +def main(): + args = get_args() + config = read_config(args.file) + + if 'openstack_translations' in config: + translate_options = get_translate_options(config, args.target) + print(' '.join(translate_options)) + return + + modules = get_valid_modules(config, args.project, args.target) + + if modules: + print(' '.join(modules)) + +if __name__ == '__main__': + main() diff --git a/roles/prepare-zanata-client/files/propose_translation_update.sh b/roles/prepare-zanata-client/files/propose_translation_update.sh new file mode 100755 index 00000000..41cafa87 --- /dev/null +++ b/roles/prepare-zanata-client/files/propose_translation_update.sh @@ -0,0 +1,357 @@ +#!/bin/bash -xe + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +PROJECT=$1 +BRANCH=$2 +JOBNAME=$3 + +# Replace /'s in the branch name with -'s because Zanata does not +# allow /'s in version names. +ZANATA_VERSION=${BRANCH//\//-} + +SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $SCRIPTSDIR/common_translation_update.sh + +init_branch $BRANCH + +function cleanup_module { + local modulename=$1 + + # Remove obsolete files. + cleanup_po_files "$modulename" + cleanup_pot_files "$modulename" + + # Compress downloaded po files, this needs to be done after + # cleanup_po_files since that function needs to have information the + # number of untranslated strings. + compress_po_files "$modulename" +} + +# Add all po files to the git repo in a target directory +function git_add_po_files { + local target_dir=$1 + + local po_file_count + + po_file_count=`find $1 -name *.po | wc -l` + + if [ $po_file_count -ne 0 ]; then + git add $target_dir/*/* + fi +} + +# Add all JSON files to the git repo (used in javascript translations) +function git_add_json_files { + local target_dir=$1 + + local json_file_count + + json_file_count=`find $1 -name '*.json' | wc -l` + + if [ $json_file_count -ne 0 ]; then + git add $target_dir/* + fi +} + +# Propose updates for manuals +function propose_manuals { + + # Pull updated translations from Zanata. + pull_from_zanata "$PROJECT" + + # Compress downloaded po files + case "$PROJECT" in + openstack-manuals) + # Cleanup po and pot files + cleanup_module "doc" + ;; + api-site) + # Cleanup po and pot files + cleanup_module "api-quick-start" + cleanup_module "firstapp" + ;; + security-doc) + cleanup_module "security-guide" + ;; + esac + + # Add imported upstream translations to git + for FILE in ${DocFolder}/*; do + DOCNAME=${FILE#${DocFolder}/} + if [ -d ${DocFolder}/${DOCNAME}/locale ] ; then + git_add_po_files ${DocFolder}/${DOCNAME}/locale + fi + if [ -d ${DocFolder}/${DOCNAME}/source/locale ] ; then + git_add_po_files ${DocFolder}/${DOCNAME}/source/locale + fi + done +} + +# Propose updates for training-guides +function propose_training_guides { + + # Pull updated translations from Zanata. + pull_from_zanata "$PROJECT" + + # Cleanup po and pot files + cleanup_module "doc/upstream-training" + + # Add all changed files to git + git_add_po_files doc/upstream-training/source/locale +} + +# Propose updates for i18n +function propose_i18n { + + # Pull updated translations from Zanata. + pull_from_zanata "$PROJECT" + + # Cleanup po and pot files + cleanup_module "doc" + + # Add all changed files to git + git_add_po_files doc/source/locale +} + +# Propose updates for python and django projects +function propose_python_django { + local modulename=$1 + local version=$2 + + # Check for empty directory and exit early + local content + + content=$(ls -A $modulename/locale/) + + if [[ "$content" == "" ]] ; then + return + fi + + # Now add all changed files to git. + # Note we add them here to not have to differentiate in the functions + # between new files and files already under git control. + git_add_po_files $modulename/locale + + # Cleanup po and pot files + cleanup_module "$modulename" + if [ "$version" == "master" ] ; then + # Remove not anymore translated log files on master, but not + # on released stable branches. + cleanup_log_files "$modulename" + fi + + # Check first whether directory exists, it might be missing if + # there are no translations. + if [[ -d "$modulename/locale/" ]] ; then + + # Some files were changed, add changed files again to git, so + # that we can run git diff properly. + git_add_po_files $modulename/locale + fi +} + + +# Handle either python or django proposals +function handle_python_django_project { + local project=$1 + + setup_project "$project" "$ZANATA_VERSION" + pull_from_zanata "$project" + handle_python_django $project python + handle_python_django $project django + handle_project_doc $project +} + +# Handle project doc proposals +function handle_project_doc { + local project=$1 + # doing only things in the test repos for project doc translation + if ! [[ "$project" =~ ^(horizon|openstack-ansible|openstack-helm)$ ]]; then + return + fi + # setup_project and pull_from_zanata are already done + # we start directly with generating .pot files + extract_messages_doc + # cleanup po and pot files + cleanup_module "doc" + # Add all changed files to git + git_add_po_files doc/source/locale +} + +# Handle either python or django proposals +function handle_python_django { + local project=$1 + # kind can be "python" or "django" + local kind=$2 + local module_names + + module_names=$(get_modulename $project $kind) + if [ -n "$module_names" ]; then + if [[ "$kind" == "django" ]] ; then + install_horizon + fi + propose_releasenotes "$ZANATA_VERSION" + for modulename in $module_names; do + # Note that we need to generate the pot files so that we + # can calculate how many strings are translated. + case "$kind" in + django) + # Update the .pot file + extract_messages_django "$modulename" + ;; + python) + # Extract messages from project except log messages + extract_messages_python "$modulename" + ;; + esac + propose_python_django "$modulename" "$ZANATA_VERSION" + done + fi +} + + +function propose_releasenotes { + local version=$1 + + # This function does not check whether releasenote publishing and + # testing are set up in zuul/layout.yaml. If releasenotes exist, + # they get pushed to the translation server. + + # Note that releasenotes only get translated on master. + if [[ "$version" == "master" && -f releasenotes/source/conf.py ]]; then + + # Note that we need to generate these so that we can calculate + # how many strings are translated. + extract_messages_releasenotes "keep_workdir" + + local lang_po + local locale_dir=releasenotes/source/locale + for lang_po in $(find $locale_dir -name 'releasenotes.po'); do + check_releasenotes_per_language $lang_po + done + + # Remove the working directory. We no longer needs it. + rm -rf releasenotes/work + + # Cleanup POT files. + # PO files are already clean up in check_releasenotes_translations. + cleanup_pot_files "releasenotes" + + # Compress downloaded po files, this needs to be done after + # cleanup_po_files since that function needs to have information the + # number of untranslated strings. + compress_po_files "releasenotes" + + # Add all changed files to git - if there are + # translated files at all. + if [ -d releasenotes/source/locale/ ] ; then + git_add_po_files releasenotes/source/locale + fi + fi + + # Remove any releasenotes translations from stable branches, they + # are not needed there. + if [[ "$version" != "master" && -d releasenotes/source/locale ]]; then + # Note that content might exist, e.g. from downloaded translations, + # but are not under git control. + git rm --ignore-unmatch -rf releasenotes/source/locale + fi +} + + +function propose_reactjs { + pull_from_zanata "$PROJECT" + + # Clean up files (removes incomplete translations and untranslated strings) + cleanup_module "i18n" + + # Convert po files to ReactJS i18n JSON format + for lang in `find i18n/*.po -printf "%f\n" | sed 's/\.po$//'`; do + npm run po2json -- ./i18n/$lang.po -o ./i18n/$lang.json + # The files are created as a one-line JSON file - expand them + python -m json.tool ./i18n/$lang.json ./i18n/locales/$lang.json + rm ./i18n/$lang.json + done + + # Add JSON files to git + git_add_json_files i18n/locales +} + + +# Setup git repository for git review. +setup_git + +# Check whether a review already exists, setup review commit message. +# Function setup_review calls setup_commit_message which will set CHANGE_ID and +# CHANGE_NUM if a change exists and will always set COMMIT_MSG. +setup_review "$BRANCH" + +# If a change already exists, let's pull it in and compute the +# 'git patch-id' of it. +PREV_PATCH_ID="" +if [[ -n ${CHANGE_NUM} ]]; then + # Ignore errors we get in case we can't download the patch with + # git-review. If that happens then we will submit a new patch. + set +e + git review -d ${CHANGE_NUM} + RET=$? + if [[ "$RET" -eq 0 ]]; then + PREV_PATCH_ID=$(git show | git patch-id | awk '{print $1}') + fi + set -e + # The git review changed our branch, go back to our correct branch + git checkout -f ${BRANCH} +fi + +# Setup venv - needed for all projects for subunit +setup_venv + +case "$PROJECT" in + api-site|openstack-manuals|security-doc) + init_manuals "$PROJECT" + setup_manuals "$PROJECT" "$ZANATA_VERSION" + propose_manuals + propose_releasenotes "$ZANATA_VERSION" + ;; + training-guides) + setup_training_guides "$ZANATA_VERSION" + propose_training_guides + ;; + i18n) + setup_i18n "$ZANATA_VERSION" + propose_i18n + ;; + tripleo-ui) + setup_reactjs_project "$PROJECT" "$ZANATA_VERSION" + propose_reactjs + ;; + *) + # Common setup for python and django repositories + handle_python_django_project $PROJECT + ;; +esac + +# Filter out commits we do not want. +filter_commits + +# Propose patch to gerrit if there are changes. +send_patch "$BRANCH" + +if [ $INVALID_PO_FILE -eq 1 ] ; then + echo "At least one po file in invalid. Fix all invalid files on the" + echo "translation server." + exit 1 +fi +# Tell finish function that everything is fine. +ERROR_ABORT=0 diff --git a/roles/prepare-zanata-client/files/query-zanata-project-version.py b/roles/prepare-zanata-client/files/query-zanata-project-version.py new file mode 100755 index 00000000..6421d13b --- /dev/null +++ b/roles/prepare-zanata-client/files/query-zanata-project-version.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os +import json +import sys +from ZanataUtils import IniConfig, ZanataRestService + + +def get_args(): + parser = argparse.ArgumentParser(description='Check if a version for the ' + 'specified project exists on the Zanata ' + 'server') + parser.add_argument('-p', '--project', required=True) + parser.add_argument('-v', '--version', required=True) + parser.add_argument('--no-verify', action='store_false', dest='verify', + help='Do not perform HTTPS certificate verification') + return parser.parse_args() + + +def main(): + args = get_args() + zc = IniConfig(os.path.expanduser('~/.config/zanata.ini')) + rest_service = ZanataRestService(zc, content_type='application/json', + accept='application/json', + verify=args.verify) + try: + r = rest_service.query( + '/rest/project/%s/version/%s' + % (args.project, args.version)) + except ValueError: + sys.exit(1) + if r.status_code == 200: + details = json.loads(r.content) + if details['status'] == 'READONLY': + sys.exit(1) + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/roles/prepare-zanata-client/files/upstream_translation_update.sh b/roles/prepare-zanata-client/files/upstream_translation_update.sh new file mode 100755 index 00000000..5416032a --- /dev/null +++ b/roles/prepare-zanata-client/files/upstream_translation_update.sh @@ -0,0 +1,138 @@ +#!/bin/bash -xe + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +PROJECT=$1 +JOBNAME=$2 + +# Replace /'s in branch names with -'s because Zanata doesn't +# allow /'s in version names. +# Zuul v3 native job passes the branch in as parameter but +# does not set ZUUL_REFNAME. +if [ -z "$ZUUL_REFNAME" ] ; then + BRANCHNAME=$3 +else + BRANCHNAME=$ZUUL_REFNAME +fi +ZANATA_VERSION=${BRANCHNAME//\//-} + +SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $SCRIPTSDIR/common_translation_update.sh + +init_branch $BRANCHNAME + +# List of all modules to copy POT files from +ALL_MODULES="" + +# Setup venv - needed for all projects for our tools +setup_venv + +if ! python $SCRIPTSDIR/query-zanata-project-version.py \ + -p $PROJECT -v $ZANATA_VERSION; then + # Exit successfully so that lack of a version doesn't cause the jenkins + # jobs to fail. This is necessary because not all branches of a project + # will be translated. + + # Tell finish function that everything is fine. + ERROR_ABORT=0 + exit 0 +fi + +setup_git + +# Project setup and updating POT files. +case "$PROJECT" in + api-site|openstack-manuals|security-doc) + init_manuals "$PROJECT" + # POT file extraction is done in setup_manuals. + setup_manuals "$PROJECT" "$ZANATA_VERSION" + case "$PROJECT" in + api-site) + ALL_MODULES="api-quick-start firstapp" + ;; + security-doc) + ALL_MODULES="security-guide" + ;; + *) + ALL_MODULES="doc" + ;; + esac + if [[ "$ZANATA_VERSION" == "master" && -f releasenotes/source/conf.py ]]; then + extract_messages_releasenotes + ALL_MODULES="releasenotes $ALL_MODULES" + fi + ;; + training-guides) + setup_training_guides "$ZANATA_VERSION" + ALL_MODULES="doc" + ;; + i18n) + setup_i18n "$ZANATA_VERSION" + ALL_MODULES="doc" + ;; + tripleo-ui) + setup_reactjs_project "$PROJECT" "$ZANATA_VERSION" + # The pot file is generated in the ./i18n directory + ALL_MODULES="i18n" + ;; + *) + # Common setup for python and django repositories + setup_project "$PROJECT" "$ZANATA_VERSION" + # ---- Python projects ---- + module_names=$(get_modulename $PROJECT python) + if [ -n "$module_names" ]; then + if [[ "$ZANATA_VERSION" == "master" && -f releasenotes/source/conf.py ]]; then + extract_messages_releasenotes + ALL_MODULES="releasenotes $ALL_MODULES" + fi + for modulename in $module_names; do + extract_messages_python "$modulename" + ALL_MODULES="$modulename $ALL_MODULES" + done + fi + + # ---- Django projects ---- + module_names=$(get_modulename $PROJECT django) + if [ -n "$module_names" ]; then + install_horizon + if [[ "$ZANATA_VERSION" == "master" && -f releasenotes/source/conf.py ]]; then + extract_messages_releasenotes + ALL_MODULES="releasenotes $ALL_MODULES" + fi + for modulename in $module_names; do + extract_messages_django "$modulename" + ALL_MODULES="$modulename $ALL_MODULES" + done + fi + # ---- Documentation ---- + # Let's test this with some repos :) + DOC_TARGETS=('horizon' 'openstack-ansible' 'openstack-helm') + if [[ -f doc/source/conf.py ]]; then + if [[ ${DOC_TARGETS[*]} =~ "$PROJECT" ]]; then + extract_messages_doc + ALL_MODULES="doc $ALL_MODULES" + fi + fi + ;; +esac + +# The Zanata client works out what to send based on the zanata.xml file. +# Do not copy translations from other files for this change. +zanata-cli -B -e push --copy-trans False +# Move pot files to translation-source directory for publishing +copy_pot "$ALL_MODULES" + +mv .translation-source translation-source + +# Tell finish function that everything is fine. +ERROR_ABORT=0 diff --git a/roles/prepare-zanata-client/tasks/main.yaml b/roles/prepare-zanata-client/tasks/main.yaml new file mode 100644 index 00000000..552dbdf1 --- /dev/null +++ b/roles/prepare-zanata-client/tasks/main.yaml @@ -0,0 +1,110 @@ +--- +# This is all bad and I feel bad, but it's translated from +# https://github.com/openstack-infra/puppet-zanata/blob/master/manifests/client.pp + +- name: Find java package name + include_vars: "{{ ansible_os_family }}.yaml" + +- name: install necessary packages + package: + name: "{{ zanata_jre_package }}" + state: present + become: yes + +- name: ensure zanata install dir + file: + path: /opt/zanata + owner: "{{ ansible_ssh_user }}" + state: directory + become: true + +- name: Look for cached zanata client + stat: + path: "/opt/cache/files/zanata-cli-{{ zanata_client_version }}-dist.tar.gz" + checksum_algorithm: sha256 + register: cached_client + +- name: Ensure correct checksum of cached client + assert: + that: + - cached_client.stat.checksum == "{{ zanata_client_checksum }}" + when: cached_client.stat.exists + +- name: Extract cached client tarball + unarchive: + src: "/opt/cache/files/zanata-cli-{{ zanata_client_version }}-dist.tar.gz" + dest: /opt/zanata + creates: "/opt/zanata/zanata-cli-{{ zanata_client_version }}/bin/zanata-cli" + remote_src: yes + when: cached_client.stat.exists + +- name: Download Zanata client archive + get_url: + url: "https://search.maven.org/remotecontent?filepath=org/zanata/zanata-cli/{{ zanata_client_version }}/zanata-cli-{{ zanata_client_version }}-dist.tar.gz" + dest: "/tmp/zanata-cli-{{ zanata_client_version }}-dist.tar.gz" + checksum: "sha256:{{ zanata_client_checksum }}" + register: result + until: result | success + retries: 5 + delay: 5 + when: not cached_client.stat.exists + +- name: Extract Zanata client archive + unarchive: + src: "/tmp/zanata-cli-{{ zanata_client_version }}-dist.tar.gz" + dest: /opt/zanata + creates: "/opt/zanata/zanata-cli-{{ zanata_client_version }}/bin/zanata-cli" + remote_src: yes + when: not cached_client.stat.exists + +- name: ensure zanata-cli perms + file: + path: "/opt/zanata/zanata-cli-{{ zanata_client_version }}/bin/zanata-cli" + mode: 0755 + +- name: link zanata-cli + file: + path: /usr/local/bin/zanata-cli + src: "/opt/zanata/zanata-cli-{{ zanata_client_version }}/bin/zanata-cli" + state: link + become: true + +# This is a preview module in Ansible 2.3. It may not work. +- name: import cert to java keystore + java_cert: + cert_url: "{{ zanata_api_credentials.fqdn }}" + keystore_path: /etc/ssl/certs/java/cacerts + keystore_pass: changeit + keystore_create: true + become: true + # Use sudo to ensure root ownership + +- name: set permissions for cacert + file: + path: /etc/ssl/certs/java/cacerts + mode: 0644 + become: true + +- name: ensure zanata config dir + file: + path: ~/.config + state: directory + +- name: write out zanata config + template: + src: zanata.ini + dest: ~/.config/zanata.ini + +- name: Copy translation scripts to the script dir on the node + copy: + dest: '{{ ansible_user_dir }}/scripts/' + src: '{{ item }}' + mode: 0755 + with_items: + - common_translation_update.sh + - create-zanata-xml.py + - get-modulename.py + - propose_translation_update.sh + - query-zanata-project-version.py + - upstream_translation_update.sh + - ZanataUtils.py diff --git a/roles/prepare-zanata-client/templates/zanata.ini b/roles/prepare-zanata-client/templates/zanata.ini new file mode 100644 index 00000000..40cf6c6d --- /dev/null +++ b/roles/prepare-zanata-client/templates/zanata.ini @@ -0,0 +1,4 @@ +[servers] +{{ zanata_api_credentials.server_id }}.url={{ zanata_api_credentials.url }} +{{ zanata_api_credentials.server_id }}.username={{ zanata_api_credentials.username }} +{{ zanata_api_credentials.server_id }}.key={{ zanata_api_credentials.key }} diff --git a/roles/prepare-zanata-client/vars/Debian.yaml b/roles/prepare-zanata-client/vars/Debian.yaml new file mode 100644 index 00000000..fba0ada7 --- /dev/null +++ b/roles/prepare-zanata-client/vars/Debian.yaml @@ -0,0 +1 @@ +zanata_jre_package: default-jre-headless diff --git a/tests/base.yaml b/tests/base.yaml index 6c6a9d58..aa594c8d 100644 --- a/tests/base.yaml +++ b/tests/base.yaml @@ -11,4 +11,3 @@ - include: configure-mirrors.yaml - include: fetch-zuul-cloner.yaml - include: validate-host.yaml - diff --git a/tests/extra.yaml b/tests/extra.yaml new file mode 100644 index 00000000..8fb42670 --- /dev/null +++ b/tests/extra.yaml @@ -0,0 +1,7 @@ +# Testing for non-base roles that are used across various jobs + +# If you add new tests, also update the files section in job +# extra-integration in zuul.d/jobs.yaml. + +- include: prepare-zanata-client.yaml + when: ansible_os_family == 'Debian' diff --git a/tests/prepare-zanata-client.yaml b/tests/prepare-zanata-client.yaml new file mode 100644 index 00000000..d034b018 --- /dev/null +++ b/tests/prepare-zanata-client.yaml @@ -0,0 +1,17 @@ +- name: Test the prepare-zanata-client role + hosts: all + + vars: + zanata_api_credentials: + fqdn: translate.openstack.org + server_id: translate.openstack.org + url: https://translate.openstack.org/ + username: infra + key: a_fake_key + roles: + - role: prepare-zanata-client + + post_tasks: + - name: Check zanata client works + command: "/opt/zanata/zanata-cli-{{ zanata_client_version }}/bin/zanata-cli --version" + changed_when: false diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index a56c252b..95d54f20 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -1496,3 +1496,35 @@ nodes: - name: fedora-28 label: fedora-28-vexxhost + +- job: + name: openstack-infra-extra-integration + description: | + Runs non-base roles that are used within various jobs to prevent + regressions. As opposed to base roles, these may run in a + limited set of environments or have other simplifying + assumptions. + abstract: true + protected: true + parent: base + required-projects: + - openstack-infra/project-config + roles: + - zuul: openstack-infra/zuul-jobs + run: tests/extra.yaml + files: + - ^zuul.d/* + - ^roles/prepare-zanata-client/.* + - ^tests/.* + +# NOTE(ianw): This test restricted to the two node types these roles +# run on in the gate. +- job: + name: openstack-infra-extra-integration-xenial + parent: openstack-infra-extra-integration + nodeset: ubuntu-xenial + +- job: + name: openstack-infra-extra-integration-bionic + parent: openstack-infra-extra-integration + nodeset: ubuntu-bionic diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 8e30a8eb..002492e4 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -24,6 +24,8 @@ - openstack-infra-multinode-integration-opensuse423 - openstack-infra-multinode-integration-opensuse-tumbleweed: voting: false + - openstack-infra-extra-integration-xenial + - openstack-infra-extra-integration-bionic - openstack-zuul-jobs-linters gate: jobs: