Support pushing translations to Zanata
In order to migrate to Zanata for translations, we need to modify the proposal scripts to also upload translations to the Zanata server. This commit supports most projects (those that make use of upstream_translation_update.sh), as well as django_openstack_auth. This does not impact the existing push and pull of translations to Transifex, and also untouched are the translations for Horizon or the manuals, which are all handled in separate scripts. Co-Authored-By: stephane <stephane@alum.mit.edu> Change-Id: I3bfb188c8b0c0e65f22d7edc30721b163f084fff
This commit is contained in:
Steve Kowalik
parent
f6ab73993d
commit
9ec48c03fe
229
jenkins/scripts/ZanataUtils.py
Executable file
229
jenkins/scripts/ZanataUtils.py
Executable file
@ -0,0 +1,229 @@
|
||||
# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from io import BytesIO
|
||||
from lxml import etree
|
||||
import os
|
||||
import re
|
||||
import requests
|
||||
import sys
|
||||
try:
|
||||
import configparser
|
||||
except ImportError:
|
||||
import ConfigParser as configparser
|
||||
try:
|
||||
from urllib.parse import urljoin
|
||||
except ImportError:
|
||||
from urlparse import urljoin
|
||||
|
||||
|
||||
class IniConfig:
|
||||
"""Object that stores zanata.ini configuration
|
||||
|
||||
Read zanata.ini and make its values available.
|
||||
|
||||
Attributes:
|
||||
inifile: The path to the ini file to load values from.
|
||||
|
||||
"""
|
||||
def __init__(self, inifile):
|
||||
self.inifile = inifile
|
||||
self._load_config()
|
||||
|
||||
def _load_config(self):
|
||||
"""Load configuration from the zanata.ini file
|
||||
|
||||
Parses the ini file and stores its data.
|
||||
|
||||
"""
|
||||
if not os.path.isfile(self.inifile):
|
||||
sys.exit('zanata.ini file not found.')
|
||||
config = configparser.ConfigParser()
|
||||
try:
|
||||
config.read(self.inifile)
|
||||
except configparser.Error:
|
||||
sys.exit('zanata.ini could not be parsed, please check format.')
|
||||
for item in config.items('servers'):
|
||||
item_type = item[0].split('.')[1]
|
||||
if item_type in ('username', 'key', 'url'):
|
||||
setattr(self, item_type, item[1])
|
||||
|
||||
|
||||
class ProjectConfig:
|
||||
"""Object that stores zanata.xml per-project configuration.
|
||||
|
||||
Given an existing zanata.xml, read in the values and make
|
||||
them accessible. Otherwise, write out a zanata.xml file
|
||||
for the project given the supplied values.
|
||||
|
||||
Attributes:
|
||||
zc (IniConfig): zanata.ini values
|
||||
url (str): URL of Zanata server
|
||||
username (str): Zanata username
|
||||
key (str): Zanata API key
|
||||
xmlfile (str): path to zanata.xml to read or write
|
||||
rules (list): list of mapping rules
|
||||
|
||||
"""
|
||||
def __init__(self, zconfig, xmlfile, **kwargs):
|
||||
self.zc = zconfig
|
||||
self.url = zconfig.url
|
||||
self.username = zconfig.username
|
||||
self.key = zconfig.key
|
||||
self.xmlfile = xmlfile
|
||||
self.rules = []
|
||||
if os.path.isfile(os.path.abspath(xmlfile)):
|
||||
self._load_config()
|
||||
else:
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
self._create_config()
|
||||
|
||||
def _get_tag_prefix(self, root):
|
||||
"""XML utility method
|
||||
|
||||
Get the namespace of the XML file so we can
|
||||
use it to search for tags.
|
||||
|
||||
"""
|
||||
return '{%s}' % etree.QName(root).namespace
|
||||
|
||||
def _load_config(self):
|
||||
"""Load configuration from an existing zanata.xml
|
||||
|
||||
Load and store project configuration from zanata.xml
|
||||
|
||||
"""
|
||||
try:
|
||||
with open(self.xmlfile, 'r') as f:
|
||||
xml = etree.parse(f)
|
||||
except IOError:
|
||||
sys.exit('Cannot load zanata.xml for this project')
|
||||
except etree.ParseError:
|
||||
sys.exit('Cannot parse zanata.xml for this project')
|
||||
root = xml.getroot()
|
||||
tag_prefix = self._get_tag_prefix(root)
|
||||
self.project = root.find('%sproject' % tag_prefix).text
|
||||
self.version = root.find('%sproject-version' % tag_prefix).text
|
||||
self.srcdir = root.find('%ssrc-dir' % tag_prefix).text
|
||||
self.txdir = root.find('%strans-dir' % tag_prefix).text
|
||||
# TODO - smarter parsing of rules here
|
||||
self.rules = root.findall('%srules' % tag_prefix)
|
||||
|
||||
def _create_config(self):
|
||||
"""Create zanata.xml
|
||||
|
||||
Use the supplied parameters to create zanata.xml by downloading
|
||||
a base version of the file and adding customizations.
|
||||
|
||||
"""
|
||||
xml = self._fetch_zanata_xml()
|
||||
self._add_configuration(xml)
|
||||
self._write_xml(xml)
|
||||
|
||||
def _query_zanata_api(self, url_fragment, verify=False):
|
||||
"""Fetch a URL from Zanata
|
||||
|
||||
Attributes:
|
||||
url_fragment: A URL fragment that will be joined to the server URL.
|
||||
verify (bool): Verify the SSL certificate from the Zanata server.
|
||||
"""
|
||||
request_url = urljoin(self.url, url_fragment)
|
||||
try:
|
||||
headers = {'Accept': 'application/xml',
|
||||
'X-Auth-User': self.username,
|
||||
'X-Auth-Token': self.key}
|
||||
r = requests.get(request_url, verify=verify, headers=headers)
|
||||
except requests.exceptions.ConnectionError:
|
||||
sys.exit("Connection error")
|
||||
if r.status_code != 200:
|
||||
sys.exit('Got status code %s for %s' %
|
||||
(r.status_code, request_url))
|
||||
if not r.content:
|
||||
sys.exit('Did not recieve any data from %s' % request_url)
|
||||
return r.content
|
||||
|
||||
def _fetch_zanata_xml(self, verify=False):
|
||||
"""Get base zanata.xml
|
||||
|
||||
Download a basic version of the configuration for the project
|
||||
using Zanata's REST API.
|
||||
|
||||
Attributes:
|
||||
verify (bool): Verify the SSL certificate from the Zanata server.
|
||||
Default false.
|
||||
"""
|
||||
project_config = self._query_zanata_api(
|
||||
'/rest/projects/p/%s/iterations/i/%s/config'
|
||||
% (self.project, self.version), verify=verify)
|
||||
p = etree.XMLParser(remove_blank_text=True)
|
||||
try:
|
||||
xml = etree.parse(BytesIO(project_config), p)
|
||||
except etree.ParseError:
|
||||
sys.exit('Error parsing xml output')
|
||||
return xml
|
||||
|
||||
def _add_configuration(self, xml):
|
||||
"""
|
||||
Insert additional configuration
|
||||
|
||||
Add locale mapping rules to the base zanata.xml retrieved from
|
||||
the server.
|
||||
|
||||
Args:
|
||||
xml (etree): zanata.xml file contents
|
||||
|
||||
"""
|
||||
# TODO - need to figure out horizon dashboard as part of it is in
|
||||
# different srcdir/transdir
|
||||
root = xml.getroot()
|
||||
s = etree.SubElement(root, 'src-dir')
|
||||
s.text = self.srcdir
|
||||
t = etree.SubElement(root, 'trans-dir')
|
||||
t.text = self.txdir
|
||||
rules = etree.SubElement(root, 'rules')
|
||||
p1 = etree.SubElement(rules, 'rule')
|
||||
p1.attrib['pattern'] = '*.pot'
|
||||
p1.text = '{locale}/LC_MESSAGES/{filename}.po'
|
||||
tag_prefix = self._get_tag_prefix(root)
|
||||
locale_sub = root.find('%slocales' % tag_prefix)
|
||||
locale_elements = locale_sub.findall('%slocale' % tag_prefix)
|
||||
locales = [x.text for x in locale_elements]
|
||||
# Work out which locales are trivially mappable to the names we
|
||||
# typically use (for example, en-gb vs en_GB) and add these mappings
|
||||
# to the configuration.
|
||||
for l in locales:
|
||||
parts = l.split('-')
|
||||
if len(parts) > 1:
|
||||
parts[1] = parts[1].upper()
|
||||
e = etree.SubElement(locale_sub, 'locale')
|
||||
e.attrib['map-from'] = '_'.join(parts)
|
||||
e.text = l
|
||||
# TODO - add hardcoded mappings for additional
|
||||
# language names (for example zh-hans-*) ?
|
||||
# Work around https://bugzilla.redhat.com/show_bug.cgi?id=1219624
|
||||
# by removing port number in URL if it's there
|
||||
url = root.find('%surl' % tag_prefix)
|
||||
url.text = re.sub(':443', '', url.text)
|
||||
|
||||
def _write_xml(self, xml):
|
||||
"""Write xml
|
||||
|
||||
Write out xml to zanata.xml.
|
||||
|
||||
"""
|
||||
try:
|
||||
xml.write(self.xmlfile, pretty_print=True)
|
||||
except IOError:
|
||||
sys.exit('Error writing zanata.xml.')
|
||||
@ -34,7 +34,7 @@ function setup_translation {
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup a project for transifex
|
||||
# Setup a project for transifex or Zanata
|
||||
function setup_project {
|
||||
local project=$1
|
||||
|
||||
@ -45,6 +45,12 @@ function setup_project {
|
||||
--source-lang en \
|
||||
--source-file ${project}/locale/${project}.pot -t PO \
|
||||
--execute
|
||||
|
||||
# While we spin up, we want to not error out if we can't generate the
|
||||
# zanata.xml file.
|
||||
if ! /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project -v master --srcdir ${project}/locale --txdir ${project}/locale -f zanata.xml; then
|
||||
echo "Failed to generate zanata.xml"
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup project horizon for transifex
|
||||
@ -226,6 +232,8 @@ function send_patch {
|
||||
else
|
||||
rm -rf .tx
|
||||
fi
|
||||
# We don't have any repos storing zanata.xml, so just remove it.
|
||||
rm -f zanata.xml
|
||||
|
||||
# Don't send a review if nothing has changed.
|
||||
if [ $(git diff --cached | wc -l) -gt 0 ]; then
|
||||
@ -286,13 +294,20 @@ function extract_messages_log {
|
||||
done
|
||||
}
|
||||
|
||||
# Setup project django_openstack_auth for transifex
|
||||
# Setup project django_openstack_auth for transifex and Zanata
|
||||
function setup_django_openstack_auth {
|
||||
tx set --auto-local -r horizon.djangopo \
|
||||
"openstack_auth/locale/<lang>/LC_MESSAGES/django.po" \
|
||||
--source-lang en \
|
||||
--source-file openstack_auth/locale/openstack_auth.pot -t PO \
|
||||
--execute
|
||||
|
||||
# While we spin up, we want to not error out if we can't generate the
|
||||
# zanata.xml file.
|
||||
if ! /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project -v master --srcdir openstack_auth/locale --txdir openstack_auth/locale -f zanata.xml; then
|
||||
echo "Failed to generate zanata.xml"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
# Filter out files that we do not want to commit
|
||||
|
||||
41
jenkins/scripts/create-zanata-xml.py
Executable file
41
jenkins/scripts/create-zanata-xml.py
Executable file
@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from ZanataUtils import IniConfig, ProjectConfig
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser(description='Generate a zanata.xml '
|
||||
'file for this project so we can '
|
||||
'process translations')
|
||||
parser.add_argument('-p', '--project')
|
||||
parser.add_argument('-v', '--version')
|
||||
parser.add_argument('-s', '--srcdir')
|
||||
parser.add_argument('-d', '--txdir')
|
||||
parser.add_argument('-f', '--file', required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
zc = IniConfig(os.path.expanduser('~/.config/zanata.ini'))
|
||||
ProjectConfig(zc, args.file, project=args.project,
|
||||
version=args.version,
|
||||
srcdir=args.srcdir, txdir=args.txdir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -36,4 +36,9 @@ git add openstack_auth/locale/*
|
||||
if ! git diff-index --quiet HEAD --; then
|
||||
# Push .pot changes to transifex
|
||||
tx --debug --traceback push -s
|
||||
|
||||
# And zanata, if we have an XML file.
|
||||
if [ -f zanata.xml ]; then
|
||||
zanata-cli -B -e push
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -47,4 +47,9 @@ if ! git diff-index --quiet HEAD --; then
|
||||
-r ${tx_project}.${tx_project}-log-${level}-translations
|
||||
fi
|
||||
done
|
||||
# The Zanata client works out what to send based on the XML file, push if
|
||||
# we have one.
|
||||
if [ -f zanata.xml ]; then
|
||||
zanata-cli -B -e push
|
||||
fi
|
||||
fi
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user