From 9ec48c03fee4b7df2ab14aa62b1faf96e6cf9173 Mon Sep 17 00:00:00 2001 From: Steve Kowalik Date: Wed, 20 May 2015 11:34:57 -0700 Subject: [PATCH] Support pushing translations to Zanata In order to migrate to Zanata for translations, we need to modify the proposal scripts to also upload translations to the Zanata server. This commit supports most projects (those that make use of upstream_translation_update.sh), as well as django_openstack_auth. This does not impact the existing push and pull of translations to Transifex, and also untouched are the translations for Horizon or the manuals, which are all handled in separate scripts. Co-Authored-By: stephane Change-Id: I3bfb188c8b0c0e65f22d7edc30721b163f084fff --- jenkins/scripts/ZanataUtils.py | 229 ++++++++++++++++++ jenkins/scripts/common_translation_update.sh | 19 +- jenkins/scripts/create-zanata-xml.py | 41 ++++ ...tream_translation_django_openstack_auth.sh | 5 + .../scripts/upstream_translation_update.sh | 5 + 5 files changed, 297 insertions(+), 2 deletions(-) create mode 100755 jenkins/scripts/ZanataUtils.py create mode 100755 jenkins/scripts/create-zanata-xml.py diff --git a/jenkins/scripts/ZanataUtils.py b/jenkins/scripts/ZanataUtils.py new file mode 100755 index 0000000000..07241d27cd --- /dev/null +++ b/jenkins/scripts/ZanataUtils.py @@ -0,0 +1,229 @@ +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from io import BytesIO +from lxml import etree +import os +import re +import requests +import sys +try: + import configparser +except ImportError: + import ConfigParser as configparser +try: + from urllib.parse import urljoin +except ImportError: + from urlparse import urljoin + + +class IniConfig: + """Object that stores zanata.ini configuration + + Read zanata.ini and make its values available. + + Attributes: + inifile: The path to the ini file to load values from. + + """ + def __init__(self, inifile): + self.inifile = inifile + self._load_config() + + def _load_config(self): + """Load configuration from the zanata.ini file + + Parses the ini file and stores its data. + + """ + if not os.path.isfile(self.inifile): + sys.exit('zanata.ini file not found.') + config = configparser.ConfigParser() + try: + config.read(self.inifile) + except configparser.Error: + sys.exit('zanata.ini could not be parsed, please check format.') + for item in config.items('servers'): + item_type = item[0].split('.')[1] + if item_type in ('username', 'key', 'url'): + setattr(self, item_type, item[1]) + + +class ProjectConfig: + """Object that stores zanata.xml per-project configuration. + + Given an existing zanata.xml, read in the values and make + them accessible. Otherwise, write out a zanata.xml file + for the project given the supplied values. + + Attributes: + zc (IniConfig): zanata.ini values + url (str): URL of Zanata server + username (str): Zanata username + key (str): Zanata API key + xmlfile (str): path to zanata.xml to read or write + rules (list): list of mapping rules + + """ + def __init__(self, zconfig, xmlfile, **kwargs): + self.zc = zconfig + self.url = zconfig.url + self.username = zconfig.username + self.key = zconfig.key + self.xmlfile = xmlfile + self.rules = [] + if os.path.isfile(os.path.abspath(xmlfile)): + self._load_config() + else: + for key, value in kwargs.items(): + setattr(self, key, value) + self._create_config() + + def _get_tag_prefix(self, root): + """XML utility method + + Get the namespace of the XML file so we can + use it to search for tags. + + """ + return '{%s}' % etree.QName(root).namespace + + def _load_config(self): + """Load configuration from an existing zanata.xml + + Load and store project configuration from zanata.xml + + """ + try: + with open(self.xmlfile, 'r') as f: + xml = etree.parse(f) + except IOError: + sys.exit('Cannot load zanata.xml for this project') + except etree.ParseError: + sys.exit('Cannot parse zanata.xml for this project') + root = xml.getroot() + tag_prefix = self._get_tag_prefix(root) + self.project = root.find('%sproject' % tag_prefix).text + self.version = root.find('%sproject-version' % tag_prefix).text + self.srcdir = root.find('%ssrc-dir' % tag_prefix).text + self.txdir = root.find('%strans-dir' % tag_prefix).text + # TODO - smarter parsing of rules here + self.rules = root.findall('%srules' % tag_prefix) + + def _create_config(self): + """Create zanata.xml + + Use the supplied parameters to create zanata.xml by downloading + a base version of the file and adding customizations. + + """ + xml = self._fetch_zanata_xml() + self._add_configuration(xml) + self._write_xml(xml) + + def _query_zanata_api(self, url_fragment, verify=False): + """Fetch a URL from Zanata + + Attributes: + url_fragment: A URL fragment that will be joined to the server URL. + verify (bool): Verify the SSL certificate from the Zanata server. + """ + request_url = urljoin(self.url, url_fragment) + try: + headers = {'Accept': 'application/xml', + 'X-Auth-User': self.username, + 'X-Auth-Token': self.key} + r = requests.get(request_url, verify=verify, headers=headers) + except requests.exceptions.ConnectionError: + sys.exit("Connection error") + if r.status_code != 200: + sys.exit('Got status code %s for %s' % + (r.status_code, request_url)) + if not r.content: + sys.exit('Did not recieve any data from %s' % request_url) + return r.content + + def _fetch_zanata_xml(self, verify=False): + """Get base zanata.xml + + Download a basic version of the configuration for the project + using Zanata's REST API. + + Attributes: + verify (bool): Verify the SSL certificate from the Zanata server. + Default false. + """ + project_config = self._query_zanata_api( + '/rest/projects/p/%s/iterations/i/%s/config' + % (self.project, self.version), verify=verify) + p = etree.XMLParser(remove_blank_text=True) + try: + xml = etree.parse(BytesIO(project_config), p) + except etree.ParseError: + sys.exit('Error parsing xml output') + return xml + + def _add_configuration(self, xml): + """ + Insert additional configuration + + Add locale mapping rules to the base zanata.xml retrieved from + the server. + + Args: + xml (etree): zanata.xml file contents + + """ + # TODO - need to figure out horizon dashboard as part of it is in + # different srcdir/transdir + root = xml.getroot() + s = etree.SubElement(root, 'src-dir') + s.text = self.srcdir + t = etree.SubElement(root, 'trans-dir') + t.text = self.txdir + rules = etree.SubElement(root, 'rules') + p1 = etree.SubElement(rules, 'rule') + p1.attrib['pattern'] = '*.pot' + p1.text = '{locale}/LC_MESSAGES/{filename}.po' + tag_prefix = self._get_tag_prefix(root) + locale_sub = root.find('%slocales' % tag_prefix) + locale_elements = locale_sub.findall('%slocale' % tag_prefix) + locales = [x.text for x in locale_elements] + # Work out which locales are trivially mappable to the names we + # typically use (for example, en-gb vs en_GB) and add these mappings + # to the configuration. + for l in locales: + parts = l.split('-') + if len(parts) > 1: + parts[1] = parts[1].upper() + e = etree.SubElement(locale_sub, 'locale') + e.attrib['map-from'] = '_'.join(parts) + e.text = l + # TODO - add hardcoded mappings for additional + # language names (for example zh-hans-*) ? + # Work around https://bugzilla.redhat.com/show_bug.cgi?id=1219624 + # by removing port number in URL if it's there + url = root.find('%surl' % tag_prefix) + url.text = re.sub(':443', '', url.text) + + def _write_xml(self, xml): + """Write xml + + Write out xml to zanata.xml. + + """ + try: + xml.write(self.xmlfile, pretty_print=True) + except IOError: + sys.exit('Error writing zanata.xml.') diff --git a/jenkins/scripts/common_translation_update.sh b/jenkins/scripts/common_translation_update.sh index 64e02cd7fb..9138a26bfb 100644 --- a/jenkins/scripts/common_translation_update.sh +++ b/jenkins/scripts/common_translation_update.sh @@ -34,7 +34,7 @@ function setup_translation { fi } -# Setup a project for transifex +# Setup a project for transifex or Zanata function setup_project { local project=$1 @@ -45,6 +45,12 @@ function setup_project { --source-lang en \ --source-file ${project}/locale/${project}.pot -t PO \ --execute + + # While we spin up, we want to not error out if we can't generate the + # zanata.xml file. + if ! /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project -v master --srcdir ${project}/locale --txdir ${project}/locale -f zanata.xml; then + echo "Failed to generate zanata.xml" + fi } # Setup project horizon for transifex @@ -226,6 +232,8 @@ function send_patch { else rm -rf .tx fi + # We don't have any repos storing zanata.xml, so just remove it. + rm -f zanata.xml # Don't send a review if nothing has changed. if [ $(git diff --cached | wc -l) -gt 0 ]; then @@ -286,13 +294,20 @@ function extract_messages_log { done } -# Setup project django_openstack_auth for transifex +# Setup project django_openstack_auth for transifex and Zanata function setup_django_openstack_auth { tx set --auto-local -r horizon.djangopo \ "openstack_auth/locale//LC_MESSAGES/django.po" \ --source-lang en \ --source-file openstack_auth/locale/openstack_auth.pot -t PO \ --execute + + # While we spin up, we want to not error out if we can't generate the + # zanata.xml file. + if ! /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project -v master --srcdir openstack_auth/locale --txdir openstack_auth/locale -f zanata.xml; then + echo "Failed to generate zanata.xml" + fi + } # Filter out files that we do not want to commit diff --git a/jenkins/scripts/create-zanata-xml.py b/jenkins/scripts/create-zanata-xml.py new file mode 100755 index 0000000000..f15cfc6597 --- /dev/null +++ b/jenkins/scripts/create-zanata-xml.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os +from ZanataUtils import IniConfig, ProjectConfig + + +def get_args(): + parser = argparse.ArgumentParser(description='Generate a zanata.xml ' + 'file for this project so we can ' + 'process translations') + parser.add_argument('-p', '--project') + parser.add_argument('-v', '--version') + parser.add_argument('-s', '--srcdir') + parser.add_argument('-d', '--txdir') + parser.add_argument('-f', '--file', required=True) + return parser.parse_args() + + +def main(): + args = get_args() + zc = IniConfig(os.path.expanduser('~/.config/zanata.ini')) + ProjectConfig(zc, args.file, project=args.project, + version=args.version, + srcdir=args.srcdir, txdir=args.txdir) + + +if __name__ == '__main__': + main() diff --git a/jenkins/scripts/upstream_translation_django_openstack_auth.sh b/jenkins/scripts/upstream_translation_django_openstack_auth.sh index deaa66c3b2..6dd396a8fa 100755 --- a/jenkins/scripts/upstream_translation_django_openstack_auth.sh +++ b/jenkins/scripts/upstream_translation_django_openstack_auth.sh @@ -36,4 +36,9 @@ git add openstack_auth/locale/* if ! git diff-index --quiet HEAD --; then # Push .pot changes to transifex tx --debug --traceback push -s + + # And zanata, if we have an XML file. + if [ -f zanata.xml ]; then + zanata-cli -B -e push + fi fi diff --git a/jenkins/scripts/upstream_translation_update.sh b/jenkins/scripts/upstream_translation_update.sh index c981151483..b73ab0ac85 100755 --- a/jenkins/scripts/upstream_translation_update.sh +++ b/jenkins/scripts/upstream_translation_update.sh @@ -47,4 +47,9 @@ if ! git diff-index --quiet HEAD --; then -r ${tx_project}.${tx_project}-log-${level}-translations fi done + # The Zanata client works out what to send based on the XML file, push if + # we have one. + if [ -f zanata.xml ]; then + zanata-cli -B -e push + fi fi