2014-05-24 20:23:05 +02:00
|
|
|
#!/bin/bash -xe
|
|
|
|
# Common code used by propose_translation_update.sh and
|
|
|
|
# upstream_translation_update.sh
|
|
|
|
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
|
|
# not use this file except in compliance with the License. You may obtain
|
|
|
|
# a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
|
|
# License for the specific language governing permissions and limitations
|
|
|
|
# under the License.
|
|
|
|
|
2015-05-20 14:17:05 -07:00
|
|
|
source /usr/local/jenkins/slave_scripts/common.sh
|
|
|
|
|
2015-12-27 20:31:41 +01:00
|
|
|
# Topic to use for our changes
|
|
|
|
TOPIC=zanata/translations
|
|
|
|
|
2015-05-03 09:55:00 +02:00
|
|
|
# Used for setup.py babel commands
|
|
|
|
QUIET="--quiet"
|
|
|
|
|
2016-01-30 17:03:15 +01:00
|
|
|
# Have invalid files been found?
|
|
|
|
INVALID_PO_FILE=0
|
|
|
|
|
2016-01-12 21:42:49 +01:00
|
|
|
# Get a module name of a project
|
|
|
|
function get_modulename {
|
|
|
|
local project=$1
|
|
|
|
local target=$2
|
|
|
|
|
|
|
|
/usr/local/jenkins/slave_scripts/get-modulename.py \
|
|
|
|
-p $project -t $target
|
|
|
|
}
|
|
|
|
|
2015-09-18 21:29:59 +02:00
|
|
|
# Setup a project for Zanata
|
2014-09-29 13:28:19 -07:00
|
|
|
function setup_project {
|
2014-05-28 13:30:21 +02:00
|
|
|
local project=$1
|
2015-09-21 09:37:30 -07:00
|
|
|
local version=${2:-master}
|
2014-05-24 20:23:05 +02:00
|
|
|
|
2015-08-28 14:37:16 +10:00
|
|
|
/usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project \
|
2015-09-21 09:37:30 -07:00
|
|
|
-v $version --srcdir ${project}/locale --txdir ${project}/locale \
|
2015-08-28 14:37:16 +10:00
|
|
|
-f zanata.xml
|
2014-05-24 20:23:05 +02:00
|
|
|
}
|
|
|
|
|
2015-09-18 21:34:22 +02:00
|
|
|
# Setup project horizon for Zanata
|
2014-09-29 13:28:19 -07:00
|
|
|
function setup_horizon {
|
2014-05-28 13:30:21 +02:00
|
|
|
local project=horizon
|
2015-09-21 09:37:30 -07:00
|
|
|
local version=${1:-master}
|
2014-05-28 13:30:21 +02:00
|
|
|
|
2015-08-28 14:37:16 +10:00
|
|
|
/usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project \
|
2015-09-21 09:37:30 -07:00
|
|
|
-v $version --srcdir . --txdir . -r './horizon/locale/*.pot' \
|
2015-09-03 14:46:53 +10:00
|
|
|
'horizon/locale/{locale_with_underscore}/LC_MESSAGES/{filename}.po' \
|
2015-09-09 13:07:31 +10:00
|
|
|
-r './openstack_dashboard/locale/*.pot' \
|
2015-08-20 13:39:11 +10:00
|
|
|
'openstack_dashboard/locale/{locale_with_underscore}/LC_MESSAGES/{filename}.po' \
|
2015-08-28 14:37:16 +10:00
|
|
|
-e '.*/**' -f zanata.xml
|
2014-05-28 13:30:21 +02:00
|
|
|
}
|
|
|
|
|
2014-06-01 09:18:53 +02:00
|
|
|
# Set global variable DocFolder for manuals projects
|
2014-09-29 13:28:19 -07:00
|
|
|
function init_manuals {
|
2014-06-01 09:18:53 +02:00
|
|
|
project=$1
|
|
|
|
|
|
|
|
DocFolder="doc"
|
2015-09-09 16:44:25 +10:00
|
|
|
ZanataDocFolder="./doc"
|
2014-06-30 18:00:30 +02:00
|
|
|
if [ $project = "api-site" -o $project = "security-doc" ] ; then
|
2014-06-01 09:18:53 +02:00
|
|
|
DocFolder="./"
|
2015-09-09 16:44:25 +10:00
|
|
|
ZanataDocFolder="."
|
2014-06-01 09:18:53 +02:00
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
# Setup project manuals projects (api-site, openstack-manuals,
|
2015-09-18 21:34:22 +02:00
|
|
|
# operations-guide) for Zanata
|
2014-09-29 13:28:19 -07:00
|
|
|
function setup_manuals {
|
2014-07-08 08:45:02 +02:00
|
|
|
local project=$1
|
2015-09-21 09:37:30 -07:00
|
|
|
local version=${2:-master}
|
2015-03-06 11:38:50 +01:00
|
|
|
|
|
|
|
# Fill in associative array SPECIAL_BOOKS
|
|
|
|
declare -A SPECIAL_BOOKS
|
|
|
|
source doc-tools-check-languages.conf
|
|
|
|
|
2015-06-29 17:32:48 +10:00
|
|
|
# Grab all of the rules for the documents we care about
|
|
|
|
ZANATA_RULES=
|
|
|
|
|
2015-11-23 09:06:36 +01:00
|
|
|
# List of directories to skip
|
|
|
|
if [ "$project" == "openstack-manuals" ]; then
|
|
|
|
EXCLUDE='.*/**,**/source/common/**'
|
|
|
|
else
|
|
|
|
EXCLUDE='.*/**,**/source/common/**,**/glossary/**'
|
|
|
|
fi
|
|
|
|
|
2014-06-01 09:18:53 +02:00
|
|
|
# Generate pot one by one
|
2014-09-29 13:28:19 -07:00
|
|
|
for FILE in ${DocFolder}/*; do
|
2014-06-01 09:18:53 +02:00
|
|
|
# Skip non-directories
|
2014-09-29 13:28:19 -07:00
|
|
|
if [ ! -d $FILE ]; then
|
2014-06-01 09:18:53 +02:00
|
|
|
continue
|
|
|
|
fi
|
|
|
|
DOCNAME=${FILE#${DocFolder}/}
|
|
|
|
# Ignore directories that will not get translated
|
2014-11-09 10:18:58 +01:00
|
|
|
if [[ "$DOCNAME" =~ ^(www|tools|generated|publish-docs)$ ]]; then
|
|
|
|
continue
|
|
|
|
fi
|
2014-07-08 08:45:02 +02:00
|
|
|
# Skip glossary in all repos besides openstack-manuals.
|
2014-09-29 13:28:19 -07:00
|
|
|
if [ "$project" != "openstack-manuals" -a "$DOCNAME" == "glossary" ]; then
|
2014-07-08 08:45:02 +02:00
|
|
|
continue
|
|
|
|
fi
|
2015-03-06 11:38:50 +01:00
|
|
|
IS_RST=0
|
|
|
|
if [ ${SPECIAL_BOOKS["${DOCNAME}"]+_} ] ; then
|
2015-03-09 18:37:06 +01:00
|
|
|
case "${SPECIAL_BOOKS["${DOCNAME}"]}" in
|
|
|
|
RST)
|
|
|
|
IS_RST=1
|
|
|
|
;;
|
|
|
|
skip)
|
2015-11-23 09:06:36 +01:00
|
|
|
EXCLUDE="$EXCLUDE,${DocFolder}/${DOCNAME}/**"
|
2015-03-09 18:37:06 +01:00
|
|
|
continue
|
|
|
|
;;
|
|
|
|
esac
|
2015-03-06 11:38:50 +01:00
|
|
|
fi
|
|
|
|
if [ ${IS_RST} -eq 1 ] ; then
|
|
|
|
tox -e generatepot-rst -- ${DOCNAME}
|
|
|
|
git add ${DocFolder}/${DOCNAME}/source/locale/${DOCNAME}.pot
|
2015-09-09 16:44:25 +10:00
|
|
|
ZANATA_RULES="$ZANATA_RULES -r ${ZanataDocFolder}/${DOCNAME}/source/locale/${DOCNAME}.pot ${DocFolder}/${DOCNAME}/source/locale/{locale_with_underscore}/LC_MESSAGES/${DOCNAME}.po"
|
2015-03-06 11:38:50 +01:00
|
|
|
else
|
|
|
|
# Update the .pot file
|
|
|
|
./tools/generatepot ${DOCNAME}
|
|
|
|
if [ -f ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot ]; then
|
|
|
|
# Add all changed files to git
|
|
|
|
git add ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot
|
2015-09-09 16:44:25 +10:00
|
|
|
ZANATA_RULES="$ZANATA_RULES -r ${ZanataDocFolder}/${DOCNAME}/locale/${DOCNAME}.pot ${DocFolder}/${DOCNAME}/locale/{locale_with_underscore}.po"
|
2015-03-06 11:38:50 +01:00
|
|
|
fi
|
2014-06-01 09:18:53 +02:00
|
|
|
fi
|
|
|
|
done
|
2015-08-28 14:37:16 +10:00
|
|
|
/usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project \
|
2015-09-21 09:37:30 -07:00
|
|
|
-v $version --srcdir . --txdir . $ZANATA_RULES -e "$EXCLUDE" \
|
2015-08-28 14:37:16 +10:00
|
|
|
-f zanata.xml
|
2014-06-01 09:18:53 +02:00
|
|
|
}
|
|
|
|
|
2015-11-16 10:43:42 +09:00
|
|
|
# Setup a training-guides project for Zanata
|
|
|
|
function setup_training_guides {
|
2015-11-24 19:04:52 +11:00
|
|
|
local project=training-guides
|
|
|
|
local version=${1:-master}
|
2015-11-16 10:43:42 +09:00
|
|
|
|
|
|
|
# Update the .pot file
|
|
|
|
tox -e generatepot-training
|
|
|
|
|
|
|
|
/usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project \
|
2015-11-24 19:04:52 +11:00
|
|
|
-v $version --srcdir doc/upstream-training/source/locale \
|
|
|
|
--txdir doc/upstream-training/source/locale \
|
2015-11-16 10:43:42 +09:00
|
|
|
-f zanata.xml
|
|
|
|
}
|
|
|
|
|
2014-05-24 20:23:05 +02:00
|
|
|
# Setup project so that git review works, sets global variable
|
|
|
|
# COMMIT_MSG.
|
2014-09-29 13:28:19 -07:00
|
|
|
function setup_review {
|
2015-09-21 10:16:58 -07:00
|
|
|
# Note we cannot rely on the default branch in .gitreview being
|
|
|
|
# correct so we are very explicit here.
|
|
|
|
local branch=${1:-master}
|
2014-08-11 08:12:21 +02:00
|
|
|
FULL_PROJECT=$(grep project .gitreview | cut -f2 -d= |sed -e 's/\.git$//')
|
2014-10-21 09:11:15 +02:00
|
|
|
set +e
|
2015-12-27 20:31:41 +01:00
|
|
|
read -d '' INITIAL_COMMIT_MSG <<EOF
|
2015-09-16 15:53:14 +10:00
|
|
|
Imported Translations from Zanata
|
2014-10-15 08:49:18 +02:00
|
|
|
|
|
|
|
For more information about this automatic import see:
|
|
|
|
https://wiki.openstack.org/wiki/Translations/Infrastructure
|
|
|
|
EOF
|
2014-10-21 09:11:15 +02:00
|
|
|
set -e
|
2014-05-24 20:23:05 +02:00
|
|
|
git review -s
|
|
|
|
|
2015-09-11 10:28:59 -07:00
|
|
|
# See if there is an open change in the zanata/translations
|
2014-05-24 20:23:05 +02:00
|
|
|
# topic. If so, get the change id for the existing change for use
|
|
|
|
# in the commit msg.
|
2015-12-27 20:45:45 +01:00
|
|
|
# Function setup_commit_message will set CHANGE_ID if a change
|
|
|
|
# exists and will always set COMMIT_MSG.
|
2015-12-27 20:31:41 +01:00
|
|
|
setup_commit_message $FULL_PROJECT proposal-bot $branch $TOPIC "$INITIAL_COMMIT_MSG"
|
2014-10-15 08:49:18 +02:00
|
|
|
|
2014-09-11 16:45:50 +02:00
|
|
|
# If the open change an already approved, let's not queue a new
|
|
|
|
# patch but let's merge the other patch first.
|
|
|
|
# This solves the problem that when the gate pipeline backup
|
|
|
|
# reaches roughly a day, no matter how quickly you approve the new
|
|
|
|
# update it will always get sniped out of the gate by another.
|
|
|
|
# It also helps, when you approve close to the time this job is
|
|
|
|
# run.
|
2015-12-27 20:31:41 +01:00
|
|
|
if [ -n "$CHANGE_ID" ]; then
|
2014-09-11 16:45:50 +02:00
|
|
|
# Use the JSON format since it is very compact and easy to grep
|
2015-12-27 20:31:41 +01:00
|
|
|
change_info=$(ssh -p 29418 proposal-bot@review.openstack.org gerrit query --current-patch-set --format=JSON $CHANGE_ID)
|
2014-09-11 16:45:50 +02:00
|
|
|
# Check for:
|
|
|
|
# 1) Workflow approval (+1)
|
|
|
|
# 2) no -1/-2 by Jenkins
|
|
|
|
# 3) no -2 by reviewers
|
2014-09-17 08:22:17 +02:00
|
|
|
# 4) no Workflow -1 (WIP)
|
2014-09-11 16:45:50 +02:00
|
|
|
#
|
2015-06-04 15:34:52 +10:00
|
|
|
if echo $change_info|grep -q '{"type":"Workflow","description":"Workflow","value":"1"' \
|
|
|
|
&& ! echo $change_info|grep -q '{"type":"Verified","description":"Verified","value":"-[12]","grantedOn":[0-9]*,"by":{"name":"Jenkins","username":"jenkins"}}' \
|
|
|
|
&& ! echo $change_info|grep -q '{"type":"Code-Review","description":"Code-Review","value":"-2"' \
|
|
|
|
&& ! echo $change_info|grep -q '{"type":"Workflow","description":"Workflow","value":"-1"' ; then
|
2014-09-11 16:45:50 +02:00
|
|
|
echo "Job already approved, exiting"
|
|
|
|
exit 0
|
|
|
|
fi
|
|
|
|
fi
|
2014-05-24 20:23:05 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
# Propose patch using COMMIT_MSG
|
2014-09-29 13:28:19 -07:00
|
|
|
function send_patch {
|
2015-09-21 10:16:58 -07:00
|
|
|
local branch=${1:-master}
|
2015-10-23 08:14:48 +02:00
|
|
|
local output
|
|
|
|
local ret
|
|
|
|
local success=0
|
2014-03-23 21:37:09 +01:00
|
|
|
|
2015-05-20 11:34:57 -07:00
|
|
|
# We don't have any repos storing zanata.xml, so just remove it.
|
|
|
|
rm -f zanata.xml
|
2014-03-23 21:37:09 +01:00
|
|
|
|
2014-05-24 20:23:05 +02:00
|
|
|
# Don't send a review if nothing has changed.
|
2015-06-04 15:33:48 +10:00
|
|
|
if [ $(git diff --cached | wc -l) -gt 0 ]; then
|
2014-05-24 20:23:05 +02:00
|
|
|
# Commit and review
|
|
|
|
git commit -F- <<EOF
|
|
|
|
$COMMIT_MSG
|
|
|
|
EOF
|
2015-10-23 08:14:48 +02:00
|
|
|
# Do error checking manually to ignore one class of failure.
|
|
|
|
set +e
|
2015-09-21 10:16:58 -07:00
|
|
|
# We cannot rely on the default branch in .gitreview being
|
|
|
|
# correct so we are very explicit here.
|
2015-12-27 20:31:41 +01:00
|
|
|
output=$(git review -t $TOPIC $branch)
|
2015-10-23 08:14:48 +02:00
|
|
|
ret=$?
|
|
|
|
[[ "$ret" -eq "0" || "$output" =~ "No changes between prior commit" ]]
|
|
|
|
success=$?
|
|
|
|
set -e
|
2014-05-24 20:23:05 +02:00
|
|
|
fi
|
2015-10-23 08:14:48 +02:00
|
|
|
return $success
|
2014-05-24 20:23:05 +02:00
|
|
|
}
|
2014-05-24 20:12:35 +02:00
|
|
|
|
|
|
|
# Setup global variables LEVELS and LKEYWORDS
|
2014-09-29 13:28:19 -07:00
|
|
|
function setup_loglevel_vars {
|
2014-05-24 20:12:35 +02:00
|
|
|
# Strings for various log levels
|
|
|
|
LEVELS="info warning error critical"
|
|
|
|
# Keywords for each log level:
|
|
|
|
declare -g -A LKEYWORD
|
|
|
|
LKEYWORD['info']='_LI'
|
|
|
|
LKEYWORD['warning']='_LW'
|
|
|
|
LKEYWORD['error']='_LE'
|
|
|
|
LKEYWORD['critical']='_LC'
|
|
|
|
}
|
|
|
|
|
2015-09-22 20:05:03 +02:00
|
|
|
# Run extract_messages for user visible messages.
|
|
|
|
function extract_messages {
|
2016-01-09 16:49:34 +01:00
|
|
|
local project=$1
|
|
|
|
|
|
|
|
# In case this is an initial run, the locale directory might not
|
|
|
|
# exist, so create it since extract_messages will fail if it does
|
|
|
|
# not exist. So, create it if needed.
|
|
|
|
mkdir -p ${project}/locale
|
2014-05-24 20:12:35 +02:00
|
|
|
|
2015-09-22 20:05:03 +02:00
|
|
|
# Update the .pot files
|
|
|
|
# The "_C" and "_P" prefix are for more-gettext-support blueprint,
|
|
|
|
# "_C" for message with context, "_P" for plural form message.
|
|
|
|
python setup.py $QUIET extract_messages --keyword "_C:1c,2 _P:1,2"
|
|
|
|
}
|
|
|
|
|
2016-01-21 16:47:32 +09:00
|
|
|
# TODO(amotoki): Finally this should replace current extract_messages.
|
|
|
|
function extract_messages_new {
|
|
|
|
local modulename=$1
|
|
|
|
|
|
|
|
# NOTE(amotoki): POT file == $modulename/locale/$modulename.pot
|
|
|
|
local POT=${modulename}/locale/${modulename}.pot
|
|
|
|
|
|
|
|
# In case this is an initial run, the locale directory might not
|
|
|
|
# exist, so create it since extract_messages will fail if it does
|
|
|
|
# not exist. So, create it if needed.
|
|
|
|
mkdir -p ${modulename}/locale
|
|
|
|
|
|
|
|
# Update the .pot files
|
|
|
|
# The "_C" and "_P" prefix are for more-gettext-support blueprint,
|
|
|
|
# "_C" for message with context, "_P" for plural form message.
|
|
|
|
python setup.py $QUIET extract_messages --keyword "_C:1c,2 _P:1,2" \
|
|
|
|
--output-file ${POT}
|
|
|
|
}
|
|
|
|
|
2015-09-22 20:05:03 +02:00
|
|
|
# Run extract_messages for log messages.
|
2014-05-24 20:12:35 +02:00
|
|
|
# Needs variables setup via setup_loglevel_vars.
|
2014-09-29 13:28:19 -07:00
|
|
|
function extract_messages_log {
|
2015-09-27 11:26:28 +02:00
|
|
|
local project=$1
|
|
|
|
local POT
|
|
|
|
local trans
|
2014-05-24 20:12:35 +02:00
|
|
|
|
|
|
|
# Update the .pot files
|
|
|
|
for level in $LEVELS ; do
|
2015-09-27 11:26:28 +02:00
|
|
|
POT=${project}/locale/${project}-log-${level}.pot
|
2015-05-03 09:55:00 +02:00
|
|
|
python setup.py $QUIET extract_messages --no-default-keywords \
|
2014-05-24 20:12:35 +02:00
|
|
|
--keyword ${LKEYWORD[$level]} \
|
2015-09-27 11:26:28 +02:00
|
|
|
--output-file ${POT}
|
|
|
|
# We don't need to add or send around empty source files.
|
|
|
|
trans=$(msgfmt --statistics -o /dev/null ${POT} 2>&1)
|
|
|
|
if [ "$trans" = "0 translated messages." ] ; then
|
|
|
|
rm $POT
|
|
|
|
# Remove file from git if it's under version control.
|
|
|
|
git rm --ignore-unmatch $POT
|
|
|
|
fi
|
2014-05-24 20:12:35 +02:00
|
|
|
done
|
|
|
|
}
|
2014-05-28 12:33:01 +02:00
|
|
|
|
2016-01-21 16:47:32 +09:00
|
|
|
# TODO(amotoki): Finally this should replace current extract_messages_log.
|
|
|
|
function extract_messages_log_new {
|
|
|
|
local modulename=$1
|
|
|
|
local POT
|
|
|
|
local trans
|
|
|
|
|
|
|
|
# Update the .pot files
|
|
|
|
for level in $LEVELS ; do
|
|
|
|
# NOTE(amotoki): POT file == $modulename/locale/$modulename.pot
|
|
|
|
POT=${modulename}/locale/${modulename}-log-${level}.pot
|
|
|
|
python setup.py $QUIET extract_messages --no-default-keywords \
|
|
|
|
--keyword ${LKEYWORD[$level]} \
|
|
|
|
--output-file ${POT}
|
|
|
|
# We don't need to add or send around empty source files.
|
|
|
|
trans=$(msgfmt --statistics -o /dev/null ${POT} 2>&1)
|
|
|
|
if [ "$trans" = "0 translated messages." ] ; then
|
|
|
|
rm $POT
|
|
|
|
# Remove file from git if it's under version control.
|
|
|
|
git rm --ignore-unmatch $POT
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
}
|
|
|
|
|
2016-01-12 21:42:49 +01:00
|
|
|
# TODO(amotoki): After we use $modulename/locale/$modulename.pot
|
|
|
|
# for normal python projects, this function will be used both
|
|
|
|
# by python and django projects. It should be renamed to setup_project.
|
|
|
|
# Setup django project for Zanata
|
|
|
|
function setup_django {
|
|
|
|
local project=$1
|
|
|
|
local modulename=$2
|
|
|
|
local version=${3:-master}
|
|
|
|
|
|
|
|
/usr/local/jenkins/slave_scripts/create-zanata-xml.py \
|
|
|
|
-p $project -v $version --srcdir $modulename/locale \
|
|
|
|
--txdir $modulename/locale -r '**/*.pot' \
|
|
|
|
'{locale_with_underscore}/LC_MESSAGES/{filename}.po' -f zanata.xml
|
|
|
|
}
|
|
|
|
|
|
|
|
# Extract messages for a django project, we need to update django.pot
|
|
|
|
# and djangojs.pot.
|
|
|
|
function extract_messages_django {
|
|
|
|
local modulename=$1
|
|
|
|
|
|
|
|
# We need to install horizon
|
|
|
|
VENV=$(mktemp -d)
|
|
|
|
trap "rm -rf $VENV" EXIT
|
|
|
|
virtualenv $VENV
|
|
|
|
|
|
|
|
# TODO(jaegerandi): Switch to zuul-cloner once it's safe to use
|
|
|
|
# zuul-cloner in post jobs and we have a persistent cache on
|
|
|
|
# proposal node.
|
|
|
|
root=$(mktemp -d)
|
|
|
|
trap "rm -rf $VENV $root" EXIT
|
|
|
|
|
|
|
|
git clone --depth=1 git://git.openstack.org/openstack/horizon.git $root/horizon
|
|
|
|
(cd ${root}/horizon && $VENV/bin/pip install .)
|
|
|
|
|
|
|
|
# Horizon has these as dependencies but let's be sure.
|
|
|
|
# TODO(amotoki): Pull required versions from g-r.
|
|
|
|
$VENV/bin/pip install Babel django-babel
|
|
|
|
KEYWORDS="-k gettext_noop -k gettext_lazy -k ngettext_lazy:1,2"
|
|
|
|
KEYWORDS+=" -k ugettext_noop -k ugettext_lazy -k ungettext_lazy:1,2"
|
|
|
|
KEYWORDS+=" -k npgettext:1c,2,3 -k pgettext_lazy:1c,2 -k npgettext_lazy:1c,2,3"
|
|
|
|
|
|
|
|
for DOMAIN in djangojs django ; do
|
|
|
|
if [ -f babel-${DOMAIN}.cfg ]; then
|
|
|
|
mkdir -p ${modulename}/locale
|
2016-02-02 10:43:17 +01:00
|
|
|
POT=${modulename}/locale/${DOMAIN}.pot
|
|
|
|
touch ${POT}
|
2016-01-12 21:42:49 +01:00
|
|
|
$VENV/bin/pybabel extract -F babel-${DOMAIN}.cfg \
|
2016-02-02 10:43:17 +01:00
|
|
|
-o ${POT} $KEYWORDS ${modulename}
|
|
|
|
# We don't need to add or send around empty source files.
|
|
|
|
trans=$(msgfmt --statistics -o /dev/null ${POT} 2>&1)
|
|
|
|
if [ "$trans" = "0 translated messages." ] ; then
|
|
|
|
rm $POT
|
|
|
|
# Remove file from git if it's under version control.
|
|
|
|
git rm --ignore-unmatch $POT
|
|
|
|
fi
|
2016-01-12 21:42:49 +01:00
|
|
|
fi
|
|
|
|
done
|
|
|
|
rm -rf $VENV $root
|
|
|
|
trap "" EXIT
|
|
|
|
}
|
|
|
|
|
2014-05-28 12:33:01 +02:00
|
|
|
|
2016-01-30 17:03:15 +01:00
|
|
|
# Filter out files that we do not want to commit.
|
|
|
|
# Sets global variable INVALID_PO_FILE to 1 if any invalid files are
|
|
|
|
# found.
|
2014-09-29 13:28:19 -07:00
|
|
|
function filter_commits {
|
2016-01-30 17:03:15 +01:00
|
|
|
local ret
|
|
|
|
|
2014-09-20 09:06:48 +02:00
|
|
|
# Don't add new empty files.
|
2015-06-04 15:33:48 +10:00
|
|
|
for f in $(git diff --cached --name-only --diff-filter=A); do
|
2014-09-20 09:06:48 +02:00
|
|
|
# Files should have at least one non-empty msgid string.
|
2014-09-26 10:45:00 +02:00
|
|
|
if ! grep -q 'msgid "[^"]' "$f" ; then
|
2014-09-20 09:06:48 +02:00
|
|
|
git reset -q "$f"
|
|
|
|
rm "$f"
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
2014-05-28 12:33:01 +02:00
|
|
|
# Don't send files where the only things which have changed are
|
|
|
|
# the creation date, the version number, the revision date,
|
2015-05-16 09:59:33 +02:00
|
|
|
# name of last translator, comment lines, or diff file information.
|
2014-09-20 09:06:48 +02:00
|
|
|
# Also, don't send files if only .pot files would be changed.
|
2014-09-06 10:19:18 +02:00
|
|
|
PO_CHANGE=0
|
2015-05-11 09:14:21 +02:00
|
|
|
# Don't iterate over deleted files
|
2015-06-04 15:33:48 +10:00
|
|
|
for f in $(git diff --cached --name-only --diff-filter=AM); do
|
2014-07-12 09:12:33 +02:00
|
|
|
# It's ok if the grep fails
|
|
|
|
set +e
|
2015-09-26 08:51:57 +02:00
|
|
|
REGEX="(POT-Creation-Date|Project-Id-Version|PO-Revision-Date|Last-Translator|X-Generator|Generated-By)"
|
2014-05-28 12:33:01 +02:00
|
|
|
changed=$(git diff --cached "$f" \
|
2015-09-16 18:46:37 +02:00
|
|
|
| egrep -v "$REGEX" \
|
2014-05-28 12:33:01 +02:00
|
|
|
| egrep -c "^([-+][^-+#])")
|
2015-05-02 13:39:07 +02:00
|
|
|
added=$(git diff --cached "$f" \
|
2015-09-16 18:46:37 +02:00
|
|
|
| egrep -v "$REGEX" \
|
2015-05-02 13:39:07 +02:00
|
|
|
| egrep -c "^([+][^+#])")
|
2014-07-12 09:12:33 +02:00
|
|
|
set -e
|
2016-01-30 17:03:15 +01:00
|
|
|
# Check that imported po files are valid
|
|
|
|
if [[ $f =~ .po$ ]] ; then
|
|
|
|
set +e
|
|
|
|
msgfmt --check-format -o /dev/null $f
|
|
|
|
ret=$?
|
|
|
|
set -e
|
|
|
|
if [ $ret -ne "0" ] ; then
|
|
|
|
# Set change to zero so that next expression reverts
|
|
|
|
# change of this file.
|
|
|
|
changed=0
|
|
|
|
echo "ERROR: File $f is an invalid po file."
|
|
|
|
echo "ERROR: The file has not been imported and needs fixing!"
|
|
|
|
INVALID_PO_FILE=1
|
|
|
|
fi
|
|
|
|
fi
|
2014-09-29 13:28:19 -07:00
|
|
|
if [ $changed -eq 0 ]; then
|
2014-05-28 12:33:01 +02:00
|
|
|
git reset -q "$f"
|
|
|
|
git checkout -- "$f"
|
2015-05-02 13:39:07 +02:00
|
|
|
# Check for all files endig with ".po".
|
|
|
|
# We will take this import if at least one change adds new content,
|
|
|
|
# thus adding a new translation.
|
|
|
|
# If only lines are removed, we do not need this.
|
|
|
|
elif [[ $added -gt 0 && $f =~ .po$ ]] ; then
|
2014-09-20 09:06:48 +02:00
|
|
|
PO_CHANGE=1
|
2014-05-28 12:33:01 +02:00
|
|
|
fi
|
|
|
|
done
|
2014-09-06 10:19:18 +02:00
|
|
|
# If no po file was changed, only pot source files were changed
|
|
|
|
# and those changes can be ignored as they give no benefit on
|
|
|
|
# their own.
|
|
|
|
if [ $PO_CHANGE -eq 0 ] ; then
|
2015-06-20 09:26:38 +02:00
|
|
|
# New files need to be handled differently
|
|
|
|
for f in $(git diff --cached --name-only --diff-filter=A) ; do
|
|
|
|
git reset -q -- "$f"
|
|
|
|
rm "$f"
|
|
|
|
done
|
2015-06-04 15:33:48 +10:00
|
|
|
for f in $(git diff --cached --name-only) ; do
|
2015-06-16 20:08:43 +02:00
|
|
|
git reset -q -- "$f"
|
2014-09-06 10:19:18 +02:00
|
|
|
git checkout -- "$f"
|
|
|
|
done
|
|
|
|
fi
|
2014-05-28 12:33:01 +02:00
|
|
|
}
|
2014-09-26 02:00:44 +02:00
|
|
|
|
2015-09-03 12:56:24 +10:00
|
|
|
# Check the amount of translation done for a .po file, sets global variable
|
|
|
|
# RATIO.
|
|
|
|
function check_po_file {
|
|
|
|
local file=$1
|
|
|
|
local dropped_ratio=$2
|
|
|
|
|
|
|
|
trans=$(msgfmt --statistics -o /dev/null "$file" 2>&1)
|
|
|
|
check="^0 translated messages"
|
|
|
|
if [[ $trans =~ $check ]] ; then
|
|
|
|
RATIO=0
|
|
|
|
else
|
|
|
|
if [[ $trans =~ " translated message" ]] ; then
|
|
|
|
trans_no=$(echo $trans|sed -e 's/ translated message.*$//')
|
|
|
|
else
|
|
|
|
trans_no=0
|
|
|
|
fi
|
|
|
|
if [[ $trans =~ " untranslated message" ]] ; then
|
|
|
|
untrans_no=$(echo $trans|sed -e 's/^.* \([0-9]*\) untranslated message.*/\1/')
|
|
|
|
else
|
|
|
|
untrans_no=0
|
|
|
|
fi
|
|
|
|
total=$(($trans_no+$untrans_no))
|
|
|
|
RATIO=$((100*$trans_no/$total))
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2014-09-26 02:00:44 +02:00
|
|
|
# Remove obsolete files. We might have added them in the past but
|
|
|
|
# would not add them today, so let's eventually remove them.
|
|
|
|
function cleanup_po_files {
|
2016-01-21 16:47:32 +09:00
|
|
|
local modulename=$1
|
2014-09-26 02:00:44 +02:00
|
|
|
|
2016-01-21 16:47:32 +09:00
|
|
|
for i in $(find $modulename/locale -name *.po) ; do
|
2015-09-03 12:56:24 +10:00
|
|
|
check_po_file "$i"
|
|
|
|
if [ $RATIO -lt 20 ]; then
|
|
|
|
git rm -f $i
|
2014-09-26 02:00:44 +02:00
|
|
|
fi
|
|
|
|
done
|
|
|
|
}
|
2015-04-22 15:39:13 +02:00
|
|
|
|
|
|
|
# Reduce size of po files. This reduces the amount of content imported
|
|
|
|
# and makes for fewer imports.
|
|
|
|
# This does not touch the pot files. This way we can reconstruct the po files
|
|
|
|
# using "msgmerge POTFILE POFILE -o COMPLETEPOFILE".
|
|
|
|
function compress_po_files {
|
|
|
|
local directory=$1
|
|
|
|
|
2015-06-04 15:33:48 +10:00
|
|
|
for i in $(find $directory -name *.po) ; do
|
2015-05-27 16:43:11 +02:00
|
|
|
msgattrib --translated --no-location --sort-output "$i" \
|
|
|
|
--output="${i}.tmp"
|
2015-04-22 15:39:13 +02:00
|
|
|
mv "${i}.tmp" "$i"
|
|
|
|
done
|
|
|
|
}
|
2015-04-24 08:24:14 +02:00
|
|
|
|
|
|
|
# Reduce size of po files. This reduces the amount of content imported
|
|
|
|
# and makes for fewer imports.
|
|
|
|
# This does not touch the pot files. This way we can reconstruct the po files
|
|
|
|
# using "msgmerge POTFILE POFILE -o COMPLETEPOFILE".
|
|
|
|
# Give directory name to not touch files for example under .tox.
|
|
|
|
# Pass glossary flag to not touch the glossary.
|
|
|
|
function compress_manual_po_files {
|
|
|
|
local directory=$1
|
|
|
|
local glossary=$2
|
2015-06-04 15:33:48 +10:00
|
|
|
for i in $(find $directory -name *.po) ; do
|
2015-04-24 08:24:14 +02:00
|
|
|
if [ "$glossary" -eq "0" ] ; then
|
|
|
|
if [[ $i =~ "/glossary/" ]] ; then
|
|
|
|
continue
|
|
|
|
fi
|
|
|
|
fi
|
2015-05-27 16:43:11 +02:00
|
|
|
msgattrib --translated --no-location --sort-output "$i" \
|
|
|
|
--output="${i}.tmp"
|
2015-04-24 08:24:14 +02:00
|
|
|
mv "${i}.tmp" "$i"
|
|
|
|
done
|
|
|
|
}
|
2015-07-09 16:45:42 +10:00
|
|
|
|
2015-08-06 17:11:30 +10:00
|
|
|
function pull_from_zanata {
|
2015-09-03 12:56:24 +10:00
|
|
|
|
|
|
|
local project=$1
|
2015-08-06 17:11:30 +10:00
|
|
|
|
2015-09-03 12:56:24 +10:00
|
|
|
# Since Zanata does not currently have an option to not download new
|
|
|
|
# files, we download everything, and then remove new files that are not
|
|
|
|
# translated enough.
|
|
|
|
zanata-cli -B -e pull
|
|
|
|
|
2015-09-22 20:22:56 +02:00
|
|
|
|
2015-09-03 12:56:24 +10:00
|
|
|
for i in $(find . -name '*.po' ! -path './.*' -prune | cut -b3-); do
|
2015-09-22 20:22:56 +02:00
|
|
|
check_po_file "$i"
|
|
|
|
# We want new files to be >75% translated. The glossary and
|
|
|
|
# common documents in openstack-manuals have that relaxed to
|
|
|
|
# >8%.
|
2015-09-03 12:56:24 +10:00
|
|
|
percentage=75
|
|
|
|
if [ $project = "openstack-manuals" ]; then
|
|
|
|
case "$i" in
|
|
|
|
*glossary*|*common*)
|
|
|
|
percentage=8
|
|
|
|
;;
|
|
|
|
esac
|
|
|
|
fi
|
2015-09-22 20:22:56 +02:00
|
|
|
if [ $RATIO -lt $percentage ]; then
|
|
|
|
# This means the file is below the ratio, but we only want
|
|
|
|
# to delete it, if it is a new file. Files known to git
|
|
|
|
# that drop below 20% will be cleaned up by
|
|
|
|
# cleanup_po_files.
|
|
|
|
if ! git ls-files | grep -xq "$i"; then
|
2015-09-03 12:56:24 +10:00
|
|
|
rm -f "$i"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
done
|
2015-08-06 17:11:30 +10:00
|
|
|
}
|