diff --git a/jenkins/scripts/ZanataUtils.py b/jenkins/scripts/ZanataUtils.py new file mode 100755 index 0000000000..07241d27cd --- /dev/null +++ b/jenkins/scripts/ZanataUtils.py @@ -0,0 +1,229 @@ +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from io import BytesIO +from lxml import etree +import os +import re +import requests +import sys +try: + import configparser +except ImportError: + import ConfigParser as configparser +try: + from urllib.parse import urljoin +except ImportError: + from urlparse import urljoin + + +class IniConfig: + """Object that stores zanata.ini configuration + + Read zanata.ini and make its values available. + + Attributes: + inifile: The path to the ini file to load values from. + + """ + def __init__(self, inifile): + self.inifile = inifile + self._load_config() + + def _load_config(self): + """Load configuration from the zanata.ini file + + Parses the ini file and stores its data. + + """ + if not os.path.isfile(self.inifile): + sys.exit('zanata.ini file not found.') + config = configparser.ConfigParser() + try: + config.read(self.inifile) + except configparser.Error: + sys.exit('zanata.ini could not be parsed, please check format.') + for item in config.items('servers'): + item_type = item[0].split('.')[1] + if item_type in ('username', 'key', 'url'): + setattr(self, item_type, item[1]) + + +class ProjectConfig: + """Object that stores zanata.xml per-project configuration. + + Given an existing zanata.xml, read in the values and make + them accessible. Otherwise, write out a zanata.xml file + for the project given the supplied values. + + Attributes: + zc (IniConfig): zanata.ini values + url (str): URL of Zanata server + username (str): Zanata username + key (str): Zanata API key + xmlfile (str): path to zanata.xml to read or write + rules (list): list of mapping rules + + """ + def __init__(self, zconfig, xmlfile, **kwargs): + self.zc = zconfig + self.url = zconfig.url + self.username = zconfig.username + self.key = zconfig.key + self.xmlfile = xmlfile + self.rules = [] + if os.path.isfile(os.path.abspath(xmlfile)): + self._load_config() + else: + for key, value in kwargs.items(): + setattr(self, key, value) + self._create_config() + + def _get_tag_prefix(self, root): + """XML utility method + + Get the namespace of the XML file so we can + use it to search for tags. + + """ + return '{%s}' % etree.QName(root).namespace + + def _load_config(self): + """Load configuration from an existing zanata.xml + + Load and store project configuration from zanata.xml + + """ + try: + with open(self.xmlfile, 'r') as f: + xml = etree.parse(f) + except IOError: + sys.exit('Cannot load zanata.xml for this project') + except etree.ParseError: + sys.exit('Cannot parse zanata.xml for this project') + root = xml.getroot() + tag_prefix = self._get_tag_prefix(root) + self.project = root.find('%sproject' % tag_prefix).text + self.version = root.find('%sproject-version' % tag_prefix).text + self.srcdir = root.find('%ssrc-dir' % tag_prefix).text + self.txdir = root.find('%strans-dir' % tag_prefix).text + # TODO - smarter parsing of rules here + self.rules = root.findall('%srules' % tag_prefix) + + def _create_config(self): + """Create zanata.xml + + Use the supplied parameters to create zanata.xml by downloading + a base version of the file and adding customizations. + + """ + xml = self._fetch_zanata_xml() + self._add_configuration(xml) + self._write_xml(xml) + + def _query_zanata_api(self, url_fragment, verify=False): + """Fetch a URL from Zanata + + Attributes: + url_fragment: A URL fragment that will be joined to the server URL. + verify (bool): Verify the SSL certificate from the Zanata server. + """ + request_url = urljoin(self.url, url_fragment) + try: + headers = {'Accept': 'application/xml', + 'X-Auth-User': self.username, + 'X-Auth-Token': self.key} + r = requests.get(request_url, verify=verify, headers=headers) + except requests.exceptions.ConnectionError: + sys.exit("Connection error") + if r.status_code != 200: + sys.exit('Got status code %s for %s' % + (r.status_code, request_url)) + if not r.content: + sys.exit('Did not recieve any data from %s' % request_url) + return r.content + + def _fetch_zanata_xml(self, verify=False): + """Get base zanata.xml + + Download a basic version of the configuration for the project + using Zanata's REST API. + + Attributes: + verify (bool): Verify the SSL certificate from the Zanata server. + Default false. + """ + project_config = self._query_zanata_api( + '/rest/projects/p/%s/iterations/i/%s/config' + % (self.project, self.version), verify=verify) + p = etree.XMLParser(remove_blank_text=True) + try: + xml = etree.parse(BytesIO(project_config), p) + except etree.ParseError: + sys.exit('Error parsing xml output') + return xml + + def _add_configuration(self, xml): + """ + Insert additional configuration + + Add locale mapping rules to the base zanata.xml retrieved from + the server. + + Args: + xml (etree): zanata.xml file contents + + """ + # TODO - need to figure out horizon dashboard as part of it is in + # different srcdir/transdir + root = xml.getroot() + s = etree.SubElement(root, 'src-dir') + s.text = self.srcdir + t = etree.SubElement(root, 'trans-dir') + t.text = self.txdir + rules = etree.SubElement(root, 'rules') + p1 = etree.SubElement(rules, 'rule') + p1.attrib['pattern'] = '*.pot' + p1.text = '{locale}/LC_MESSAGES/{filename}.po' + tag_prefix = self._get_tag_prefix(root) + locale_sub = root.find('%slocales' % tag_prefix) + locale_elements = locale_sub.findall('%slocale' % tag_prefix) + locales = [x.text for x in locale_elements] + # Work out which locales are trivially mappable to the names we + # typically use (for example, en-gb vs en_GB) and add these mappings + # to the configuration. + for l in locales: + parts = l.split('-') + if len(parts) > 1: + parts[1] = parts[1].upper() + e = etree.SubElement(locale_sub, 'locale') + e.attrib['map-from'] = '_'.join(parts) + e.text = l + # TODO - add hardcoded mappings for additional + # language names (for example zh-hans-*) ? + # Work around https://bugzilla.redhat.com/show_bug.cgi?id=1219624 + # by removing port number in URL if it's there + url = root.find('%surl' % tag_prefix) + url.text = re.sub(':443', '', url.text) + + def _write_xml(self, xml): + """Write xml + + Write out xml to zanata.xml. + + """ + try: + xml.write(self.xmlfile, pretty_print=True) + except IOError: + sys.exit('Error writing zanata.xml.') diff --git a/jenkins/scripts/common_translation_update.sh b/jenkins/scripts/common_translation_update.sh index 64e02cd7fb..9138a26bfb 100644 --- a/jenkins/scripts/common_translation_update.sh +++ b/jenkins/scripts/common_translation_update.sh @@ -34,7 +34,7 @@ function setup_translation { fi } -# Setup a project for transifex +# Setup a project for transifex or Zanata function setup_project { local project=$1 @@ -45,6 +45,12 @@ function setup_project { --source-lang en \ --source-file ${project}/locale/${project}.pot -t PO \ --execute + + # While we spin up, we want to not error out if we can't generate the + # zanata.xml file. + if ! /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project -v master --srcdir ${project}/locale --txdir ${project}/locale -f zanata.xml; then + echo "Failed to generate zanata.xml" + fi } # Setup project horizon for transifex @@ -226,6 +232,8 @@ function send_patch { else rm -rf .tx fi + # We don't have any repos storing zanata.xml, so just remove it. + rm -f zanata.xml # Don't send a review if nothing has changed. if [ $(git diff --cached | wc -l) -gt 0 ]; then @@ -286,13 +294,20 @@ function extract_messages_log { done } -# Setup project django_openstack_auth for transifex +# Setup project django_openstack_auth for transifex and Zanata function setup_django_openstack_auth { tx set --auto-local -r horizon.djangopo \ "openstack_auth/locale//LC_MESSAGES/django.po" \ --source-lang en \ --source-file openstack_auth/locale/openstack_auth.pot -t PO \ --execute + + # While we spin up, we want to not error out if we can't generate the + # zanata.xml file. + if ! /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project -v master --srcdir openstack_auth/locale --txdir openstack_auth/locale -f zanata.xml; then + echo "Failed to generate zanata.xml" + fi + } # Filter out files that we do not want to commit diff --git a/jenkins/scripts/create-zanata-xml.py b/jenkins/scripts/create-zanata-xml.py new file mode 100755 index 0000000000..f15cfc6597 --- /dev/null +++ b/jenkins/scripts/create-zanata-xml.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os +from ZanataUtils import IniConfig, ProjectConfig + + +def get_args(): + parser = argparse.ArgumentParser(description='Generate a zanata.xml ' + 'file for this project so we can ' + 'process translations') + parser.add_argument('-p', '--project') + parser.add_argument('-v', '--version') + parser.add_argument('-s', '--srcdir') + parser.add_argument('-d', '--txdir') + parser.add_argument('-f', '--file', required=True) + return parser.parse_args() + + +def main(): + args = get_args() + zc = IniConfig(os.path.expanduser('~/.config/zanata.ini')) + ProjectConfig(zc, args.file, project=args.project, + version=args.version, + srcdir=args.srcdir, txdir=args.txdir) + + +if __name__ == '__main__': + main() diff --git a/jenkins/scripts/upstream_translation_django_openstack_auth.sh b/jenkins/scripts/upstream_translation_django_openstack_auth.sh index deaa66c3b2..6dd396a8fa 100755 --- a/jenkins/scripts/upstream_translation_django_openstack_auth.sh +++ b/jenkins/scripts/upstream_translation_django_openstack_auth.sh @@ -36,4 +36,9 @@ git add openstack_auth/locale/* if ! git diff-index --quiet HEAD --; then # Push .pot changes to transifex tx --debug --traceback push -s + + # And zanata, if we have an XML file. + if [ -f zanata.xml ]; then + zanata-cli -B -e push + fi fi diff --git a/jenkins/scripts/upstream_translation_update.sh b/jenkins/scripts/upstream_translation_update.sh index c981151483..b73ab0ac85 100755 --- a/jenkins/scripts/upstream_translation_update.sh +++ b/jenkins/scripts/upstream_translation_update.sh @@ -47,4 +47,9 @@ if ! git diff-index --quiet HEAD --; then -r ${tx_project}.${tx_project}-log-${level}-translations fi done + # The Zanata client works out what to send based on the XML file, push if + # we have one. + if [ -f zanata.xml ]; then + zanata-cli -B -e push + fi fi