Merge "Support pushing translations to Zanata"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
4ad50cfd04
229
jenkins/scripts/ZanataUtils.py
Executable file
229
jenkins/scripts/ZanataUtils.py
Executable file
@ -0,0 +1,229 @@
|
||||
# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from io import BytesIO
|
||||
from lxml import etree
|
||||
import os
|
||||
import re
|
||||
import requests
|
||||
import sys
|
||||
try:
|
||||
import configparser
|
||||
except ImportError:
|
||||
import ConfigParser as configparser
|
||||
try:
|
||||
from urllib.parse import urljoin
|
||||
except ImportError:
|
||||
from urlparse import urljoin
|
||||
|
||||
|
||||
class IniConfig:
|
||||
"""Object that stores zanata.ini configuration
|
||||
|
||||
Read zanata.ini and make its values available.
|
||||
|
||||
Attributes:
|
||||
inifile: The path to the ini file to load values from.
|
||||
|
||||
"""
|
||||
def __init__(self, inifile):
|
||||
self.inifile = inifile
|
||||
self._load_config()
|
||||
|
||||
def _load_config(self):
|
||||
"""Load configuration from the zanata.ini file
|
||||
|
||||
Parses the ini file and stores its data.
|
||||
|
||||
"""
|
||||
if not os.path.isfile(self.inifile):
|
||||
sys.exit('zanata.ini file not found.')
|
||||
config = configparser.ConfigParser()
|
||||
try:
|
||||
config.read(self.inifile)
|
||||
except configparser.Error:
|
||||
sys.exit('zanata.ini could not be parsed, please check format.')
|
||||
for item in config.items('servers'):
|
||||
item_type = item[0].split('.')[1]
|
||||
if item_type in ('username', 'key', 'url'):
|
||||
setattr(self, item_type, item[1])
|
||||
|
||||
|
||||
class ProjectConfig:
|
||||
"""Object that stores zanata.xml per-project configuration.
|
||||
|
||||
Given an existing zanata.xml, read in the values and make
|
||||
them accessible. Otherwise, write out a zanata.xml file
|
||||
for the project given the supplied values.
|
||||
|
||||
Attributes:
|
||||
zc (IniConfig): zanata.ini values
|
||||
url (str): URL of Zanata server
|
||||
username (str): Zanata username
|
||||
key (str): Zanata API key
|
||||
xmlfile (str): path to zanata.xml to read or write
|
||||
rules (list): list of mapping rules
|
||||
|
||||
"""
|
||||
def __init__(self, zconfig, xmlfile, **kwargs):
|
||||
self.zc = zconfig
|
||||
self.url = zconfig.url
|
||||
self.username = zconfig.username
|
||||
self.key = zconfig.key
|
||||
self.xmlfile = xmlfile
|
||||
self.rules = []
|
||||
if os.path.isfile(os.path.abspath(xmlfile)):
|
||||
self._load_config()
|
||||
else:
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
self._create_config()
|
||||
|
||||
def _get_tag_prefix(self, root):
|
||||
"""XML utility method
|
||||
|
||||
Get the namespace of the XML file so we can
|
||||
use it to search for tags.
|
||||
|
||||
"""
|
||||
return '{%s}' % etree.QName(root).namespace
|
||||
|
||||
def _load_config(self):
|
||||
"""Load configuration from an existing zanata.xml
|
||||
|
||||
Load and store project configuration from zanata.xml
|
||||
|
||||
"""
|
||||
try:
|
||||
with open(self.xmlfile, 'r') as f:
|
||||
xml = etree.parse(f)
|
||||
except IOError:
|
||||
sys.exit('Cannot load zanata.xml for this project')
|
||||
except etree.ParseError:
|
||||
sys.exit('Cannot parse zanata.xml for this project')
|
||||
root = xml.getroot()
|
||||
tag_prefix = self._get_tag_prefix(root)
|
||||
self.project = root.find('%sproject' % tag_prefix).text
|
||||
self.version = root.find('%sproject-version' % tag_prefix).text
|
||||
self.srcdir = root.find('%ssrc-dir' % tag_prefix).text
|
||||
self.txdir = root.find('%strans-dir' % tag_prefix).text
|
||||
# TODO - smarter parsing of rules here
|
||||
self.rules = root.findall('%srules' % tag_prefix)
|
||||
|
||||
def _create_config(self):
|
||||
"""Create zanata.xml
|
||||
|
||||
Use the supplied parameters to create zanata.xml by downloading
|
||||
a base version of the file and adding customizations.
|
||||
|
||||
"""
|
||||
xml = self._fetch_zanata_xml()
|
||||
self._add_configuration(xml)
|
||||
self._write_xml(xml)
|
||||
|
||||
def _query_zanata_api(self, url_fragment, verify=False):
|
||||
"""Fetch a URL from Zanata
|
||||
|
||||
Attributes:
|
||||
url_fragment: A URL fragment that will be joined to the server URL.
|
||||
verify (bool): Verify the SSL certificate from the Zanata server.
|
||||
"""
|
||||
request_url = urljoin(self.url, url_fragment)
|
||||
try:
|
||||
headers = {'Accept': 'application/xml',
|
||||
'X-Auth-User': self.username,
|
||||
'X-Auth-Token': self.key}
|
||||
r = requests.get(request_url, verify=verify, headers=headers)
|
||||
except requests.exceptions.ConnectionError:
|
||||
sys.exit("Connection error")
|
||||
if r.status_code != 200:
|
||||
sys.exit('Got status code %s for %s' %
|
||||
(r.status_code, request_url))
|
||||
if not r.content:
|
||||
sys.exit('Did not recieve any data from %s' % request_url)
|
||||
return r.content
|
||||
|
||||
def _fetch_zanata_xml(self, verify=False):
|
||||
"""Get base zanata.xml
|
||||
|
||||
Download a basic version of the configuration for the project
|
||||
using Zanata's REST API.
|
||||
|
||||
Attributes:
|
||||
verify (bool): Verify the SSL certificate from the Zanata server.
|
||||
Default false.
|
||||
"""
|
||||
project_config = self._query_zanata_api(
|
||||
'/rest/projects/p/%s/iterations/i/%s/config'
|
||||
% (self.project, self.version), verify=verify)
|
||||
p = etree.XMLParser(remove_blank_text=True)
|
||||
try:
|
||||
xml = etree.parse(BytesIO(project_config), p)
|
||||
except etree.ParseError:
|
||||
sys.exit('Error parsing xml output')
|
||||
return xml
|
||||
|
||||
def _add_configuration(self, xml):
|
||||
"""
|
||||
Insert additional configuration
|
||||
|
||||
Add locale mapping rules to the base zanata.xml retrieved from
|
||||
the server.
|
||||
|
||||
Args:
|
||||
xml (etree): zanata.xml file contents
|
||||
|
||||
"""
|
||||
# TODO - need to figure out horizon dashboard as part of it is in
|
||||
# different srcdir/transdir
|
||||
root = xml.getroot()
|
||||
s = etree.SubElement(root, 'src-dir')
|
||||
s.text = self.srcdir
|
||||
t = etree.SubElement(root, 'trans-dir')
|
||||
t.text = self.txdir
|
||||
rules = etree.SubElement(root, 'rules')
|
||||
p1 = etree.SubElement(rules, 'rule')
|
||||
p1.attrib['pattern'] = '*.pot'
|
||||
p1.text = '{locale}/LC_MESSAGES/{filename}.po'
|
||||
tag_prefix = self._get_tag_prefix(root)
|
||||
locale_sub = root.find('%slocales' % tag_prefix)
|
||||
locale_elements = locale_sub.findall('%slocale' % tag_prefix)
|
||||
locales = [x.text for x in locale_elements]
|
||||
# Work out which locales are trivially mappable to the names we
|
||||
# typically use (for example, en-gb vs en_GB) and add these mappings
|
||||
# to the configuration.
|
||||
for l in locales:
|
||||
parts = l.split('-')
|
||||
if len(parts) > 1:
|
||||
parts[1] = parts[1].upper()
|
||||
e = etree.SubElement(locale_sub, 'locale')
|
||||
e.attrib['map-from'] = '_'.join(parts)
|
||||
e.text = l
|
||||
# TODO - add hardcoded mappings for additional
|
||||
# language names (for example zh-hans-*) ?
|
||||
# Work around https://bugzilla.redhat.com/show_bug.cgi?id=1219624
|
||||
# by removing port number in URL if it's there
|
||||
url = root.find('%surl' % tag_prefix)
|
||||
url.text = re.sub(':443', '', url.text)
|
||||
|
||||
def _write_xml(self, xml):
|
||||
"""Write xml
|
||||
|
||||
Write out xml to zanata.xml.
|
||||
|
||||
"""
|
||||
try:
|
||||
xml.write(self.xmlfile, pretty_print=True)
|
||||
except IOError:
|
||||
sys.exit('Error writing zanata.xml.')
|
||||
@ -34,7 +34,7 @@ function setup_translation {
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup a project for transifex
|
||||
# Setup a project for transifex or Zanata
|
||||
function setup_project {
|
||||
local project=$1
|
||||
|
||||
@ -45,6 +45,12 @@ function setup_project {
|
||||
--source-lang en \
|
||||
--source-file ${project}/locale/${project}.pot -t PO \
|
||||
--execute
|
||||
|
||||
# While we spin up, we want to not error out if we can't generate the
|
||||
# zanata.xml file.
|
||||
if ! /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project -v master --srcdir ${project}/locale --txdir ${project}/locale -f zanata.xml; then
|
||||
echo "Failed to generate zanata.xml"
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup project horizon for transifex
|
||||
@ -226,6 +232,8 @@ function send_patch {
|
||||
else
|
||||
rm -rf .tx
|
||||
fi
|
||||
# We don't have any repos storing zanata.xml, so just remove it.
|
||||
rm -f zanata.xml
|
||||
|
||||
# Don't send a review if nothing has changed.
|
||||
if [ $(git diff --cached | wc -l) -gt 0 ]; then
|
||||
@ -286,13 +294,20 @@ function extract_messages_log {
|
||||
done
|
||||
}
|
||||
|
||||
# Setup project django_openstack_auth for transifex
|
||||
# Setup project django_openstack_auth for transifex and Zanata
|
||||
function setup_django_openstack_auth {
|
||||
tx set --auto-local -r horizon.djangopo \
|
||||
"openstack_auth/locale/<lang>/LC_MESSAGES/django.po" \
|
||||
--source-lang en \
|
||||
--source-file openstack_auth/locale/openstack_auth.pot -t PO \
|
||||
--execute
|
||||
|
||||
# While we spin up, we want to not error out if we can't generate the
|
||||
# zanata.xml file.
|
||||
if ! /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project -v master --srcdir openstack_auth/locale --txdir openstack_auth/locale -f zanata.xml; then
|
||||
echo "Failed to generate zanata.xml"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
# Filter out files that we do not want to commit
|
||||
|
||||
41
jenkins/scripts/create-zanata-xml.py
Executable file
41
jenkins/scripts/create-zanata-xml.py
Executable file
@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from ZanataUtils import IniConfig, ProjectConfig
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser(description='Generate a zanata.xml '
|
||||
'file for this project so we can '
|
||||
'process translations')
|
||||
parser.add_argument('-p', '--project')
|
||||
parser.add_argument('-v', '--version')
|
||||
parser.add_argument('-s', '--srcdir')
|
||||
parser.add_argument('-d', '--txdir')
|
||||
parser.add_argument('-f', '--file', required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
zc = IniConfig(os.path.expanduser('~/.config/zanata.ini'))
|
||||
ProjectConfig(zc, args.file, project=args.project,
|
||||
version=args.version,
|
||||
srcdir=args.srcdir, txdir=args.txdir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -36,4 +36,9 @@ git add openstack_auth/locale/*
|
||||
if ! git diff-index --quiet HEAD --; then
|
||||
# Push .pot changes to transifex
|
||||
tx --debug --traceback push -s
|
||||
|
||||
# And zanata, if we have an XML file.
|
||||
if [ -f zanata.xml ]; then
|
||||
zanata-cli -B -e push
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -47,4 +47,9 @@ if ! git diff-index --quiet HEAD --; then
|
||||
-r ${tx_project}.${tx_project}-log-${level}-translations
|
||||
fi
|
||||
done
|
||||
# The Zanata client works out what to send based on the XML file, push if
|
||||
# we have one.
|
||||
if [ -f zanata.xml ]; then
|
||||
zanata-cli -B -e push
|
||||
fi
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user