Ansible job definitions for OpenStack projects CI systems
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

783 lines
25 KiB

#!/bin/bash -xe
# Common code used by propose_translation_update.sh and
# upstream_translation_update.sh
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $SCRIPTSDIR/common.sh
# Set start of timestamp for subunit
TRANS_START_TIME=$(date +%s)
SUBUNIT_OUTPUT=testrepository.subunit
# Our images have os-test installed in the following location
TESTR_VENV=/usr/os-testr-env
# Topic to use for our changes
TOPIC=zanata/translations
# Used for setup.py babel commands
QUIET="--quiet"
# Have invalid files been found?
INVALID_PO_FILE=0
# ERROR_ABORT signals whether the script aborts with failure, will be
# set to 0 on successful run.
ERROR_ABORT=1
# List of repos that have doc/source translated, we test with a smaller
# set for now.
DOC_TARGETS=('contributor-guide'
'horizon'
'openstack-ansible'
'openstack-helm'
'operations-guide')
# We need a UTF-8 locale, set it properly in case it's not set.
export LANG=en_US.UTF-8
trap "finish" EXIT
# Set up some branch dependent variables
function init_branch {
local branch=$1
# The calling environment puts upper-constraints.txt in our
# working directory.
# TOX_CONSTRAINTS_FILE needs to be exported so that tox can use it
# if needed.
export TOX_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
# The deprecated name of TOX_CONSTRAINTS_FILE is UPPER_CONSTRAINTS_FILE,
# so handle that one as well for now.
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
GIT_BRANCH=$branch
}
# Get a module name of a project
function get_modulename {
local project=$1
local target=$2
python3 $SCRIPTSDIR/get-modulename.py -p $project -t $target
}
function finish {
if [[ "$ERROR_ABORT" -eq 1 ]] ; then
$TESTR_VENV/bin/generate-subunit $TRANS_START_TIME $SECONDS \
'fail' $JOBNAME >> $SUBUNIT_OUTPUT
else
$TESTR_VENV/bin/generate-subunit $TRANS_START_TIME $SECONDS \
'success' $JOBNAME >> $SUBUNIT_OUTPUT
fi
gzip -9 $SUBUNIT_OUTPUT
# Only run this if VENV is setup.
if [ "$VENV" != "" ] ; then
# Delete temporary directories
rm -rf $VENV
VENV=""
fi
}
# The ensure-babel and ensure-sphinx roles create a venv in ~/.venv containing
# the needed software. However, it's possible we may want to switch that to
# using pip install --user and ~/.local instead of a venv, so make this
# compatible with either.
function setup_venv {
if [ -d ~/.venv ] ; then
source ~/.venv/bin/activate
else
# Ensure ~/.local/bin is in the path
export PATH=~/.local/bin:$PATH
fi
}
# Setup nodejs within the python venv. Match the nodejs version with
# the one used in the nodejs6-npm jobs.
function setup_nodeenv {
# The ensure-babel and ensure-sphinx roles create a venv in
# ~/.venv containing the needed software. However, it's possible
# we may want to switch that to using pip install --user and
# ~/.local instead of a venv, so make this compatible with either.
NODE_VENV=~/.local/node_venv
if [ -d ~/.venv ] ; then
pip install nodeenv
nodeenv --node 6.9.4 $NODE_VENV
else
pip install --user nodeenv
~/.local/bin/nodeenv --node 6.9.4 $NODE_VENV
fi
source $NODE_VENV/bin/activate
}
# Setup a project for Zanata. This is used by both Python and Django projects.
# syntax: setup_project <project> <zanata_version>
function setup_project {
local project=$1
local version=$2
# Exclude all dot-files, particuarly for things such such as .tox
# and .venv
local exclude='.*/**'
python3 $SCRIPTSDIR/create-zanata-xml.py \
-p $project -v $version --srcdir . --txdir . \
-r '**/*.pot' '{path}/{locale_with_underscore}/LC_MESSAGES/{filename}.po' \
-e "$exclude" -f zanata.xml
}
# Set global variable DocFolder for manuals projects
function init_manuals {
project=$1
DocFolder="doc"
ZanataDocFolder="./doc"
if [ $project = "api-site" -o $project = "security-doc" ] ; then
DocFolder="./"
ZanataDocFolder="."
fi
}
# Setup project manuals projects (api-site, openstack-manuals,
# security-guide) for Zanata
function setup_manuals {
local project=$1
local version=${2:-master}
# Fill in associative array SPECIAL_BOOKS
declare -A SPECIAL_BOOKS
source doc-tools-check-languages.conf
# Grab all of the rules for the documents we care about
ZANATA_RULES=
# List of directories to skip.
# All manuals have a source/common subdirectory that is a symlink
# to doc/common in openstack-manuals. We have to exclude this
# source/common directory everywhere, only doc/common gets
# translated.
EXCLUDE='.*/**,**/source/common/**'
# Generate pot one by one
for FILE in ${DocFolder}/*; do
# Skip non-directories
if [ ! -d $FILE ]; then
continue
fi
DOCNAME=${FILE#${DocFolder}/}
# Ignore directories that will not get translated
if [[ "$DOCNAME" =~ ^(www|tools|generated|publish-docs)$ ]]; then
continue
fi
IS_RST=0
if [ ${SPECIAL_BOOKS["${DOCNAME}"]+_} ] ; then
case "${SPECIAL_BOOKS["${DOCNAME}"]}" in
RST)
IS_RST=1
;;
skip)
EXCLUDE="$EXCLUDE,${DocFolder}/${DOCNAME}/**"
continue
;;
esac
fi
if [ ${IS_RST} -eq 1 ] ; then
tox -e generatepot-rst -- ${DOCNAME}
ZANATA_RULES="$ZANATA_RULES -r ${ZanataDocFolder}/${DOCNAME}/source/locale/${DOCNAME}.pot ${DocFolder}/${DOCNAME}/source/locale/{locale_with_underscore}/LC_MESSAGES/${DOCNAME}.po"
else
# Update the .pot file
./tools/generatepot ${DOCNAME}
if [ -f ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot ]; then
ZANATA_RULES="$ZANATA_RULES -r ${ZanataDocFolder}/${DOCNAME}/locale/${DOCNAME}.pot ${DocFolder}/${DOCNAME}/locale/{locale_with_underscore}.po"
fi
fi
done
# Project setup and updating POT files for release notes.
if [[ $project == "openstack-manuals" ]] && [[ $version == "master" ]]; then
ZANATA_RULES="$ZANATA_RULES -r ./releasenotes/source/locale/releasenotes.pot releasenotes/source/locale/{locale_with_underscore}/LC_MESSAGES/releasenotes.po"
fi
python3 $SCRIPTSDIR/create-zanata-xml.py \
-p $project -v $version --srcdir . --txdir . \
$ZANATA_RULES -e "$EXCLUDE" \
-f zanata.xml
}
# Setup a training-guides project for Zanata
function setup_training_guides {
local project=training-guides
local version=${1:-master}
# Update the .pot file
tox -e generatepot-training
python3 $SCRIPTSDIR/create-zanata-xml.py \
-p $project -v $version \
--srcdir doc/upstream-training/source/locale \
--txdir doc/upstream-training/source/locale \
-f zanata.xml
}
# Setup a i18n project for Zanata
function setup_i18n {
local project=i18n
local version=${1:-master}
# Update the .pot file
tox -e generatepot
python3 $SCRIPTSDIR/create-zanata-xml.py \
-p $project -v $version \
--srcdir doc/source/locale \
--txdir doc/source/locale \
-f zanata.xml
}
# Setup a ReactJS project for Zanata
function setup_reactjs_project {
local project=$1
local version=$2
local exclude='node_modules/**'
setup_nodeenv
# Extract messages
npm install
npm run build
# Transform them into .pot files
npm run json2pot
python3 $SCRIPTSDIR/create-zanata-xml.py \
-p $project -v $version --srcdir . --txdir . \
-r '**/*.pot' '{path}/{locale}.po' \
-e "$exclude" -f zanata.xml
}
# Setup project so that git review works, sets global variable
# COMMIT_MSG.
function setup_review {
# Note we cannot rely on the default branch in .gitreview being
# correct so we are very explicit here.
local branch=${1:-master}
FULL_PROJECT=$(grep project .gitreview | cut -f2 -d= |sed -e 's/\.git$//')
set +e
read -d '' INITIAL_COMMIT_MSG <<EOF
Imported Translations from Zanata
For more information about this automatic import see:
https://docs.openstack.org/i18n/latest/reviewing-translation-import.html
EOF
set -e
git review -s
# See if there is an open change in the zanata/translations
# topic. If so, get the change id for the existing change for use
# in the commit msg.
# Function setup_commit_message will set CHANGE_ID if a change
# exists and will always set COMMIT_MSG.
setup_commit_message $FULL_PROJECT proposal-bot $branch $TOPIC "$INITIAL_COMMIT_MSG"
# Function check_already_approved will quit the proposal process if there
# is already an approved job with the same CHANGE_ID
check_already_approved $CHANGE_ID
}
# Propose patch using COMMIT_MSG
function send_patch {
local branch=${1:-master}
local output
local ret
local success=0
# We don't have any repos storing zanata.xml, so just remove it.
rm -f zanata.xml
# Don't send a review if nothing has changed.
if [ $(git diff --cached | wc -l) -gt 0 ]; then
# Commit and review
git commit -F- <<EOF
$COMMIT_MSG
EOF
CUR_PATCH_ID=$(git show | git patch-id | awk '{print $1}')
# Don't submit if we have the same patch id of previously submitted
# patchset
if [[ "${PREV_PATCH_ID}" != "${CUR_PATCH_ID}" ]]; then
# Show current status of tree for debugging purpose
git status
# Do error checking manually to ignore one class of failure.
set +e
# We cannot rely on the default branch in .gitreview being
# correct so we are very explicit here.
output=$(git review -t $TOPIC $branch)
ret=$?
[[ "$ret" -eq 0 || "$output" =~ "No changes between prior commit" ]]
success=$?
set -e
fi
fi
return $success
}
# Delete empty pot files
function check_empty_pot {
local pot=$1
# We don't need to add or send around empty source files.
trans=$(msgfmt --statistics -o /dev/null ${pot} 2>&1)
if [ "$trans" = "0 translated messages." ] ; then
rm $pot
# Remove file from git if it's under version control. We previously
# had all pot files under version control, so remove file also
# from git if needed.
git rm --ignore-unmatch $pot
fi
}
# Run extract_messages for python projects.
function extract_messages_python {
local modulename=$1
local pot=${modulename}/locale/${modulename}.pot
# In case this is an initial run, the locale directory might not
# exist, so create it since extract_messages will fail if it does
# not exist. So, create it if needed.
mkdir -p ${modulename}/locale
# Update the .pot files
# The "_C" and "_P" prefix are for more-gettext-support blueprint,
# "_C" for message with context, "_P" for plural form message.
pybabel ${QUIET} extract \
--add-comments Translators: \
--msgid-bugs-address="https://bugs.launchpad.net/openstack-i18n/" \
--project=${PROJECT} --version=${VERSION} \
-k "_C:1c,2" -k "_P:1,2" \
-o ${pot} ${modulename}
check_empty_pot ${pot}
}
# Django projects need horizon installed for extraction, install it in
# our venv. The function setup_venv needs to be called first.
function install_horizon {
# Zuul has checked out horizon on the same branch as the project
# already for us.
(cd ${HORIZON_DIR} &&
pip install -c $UPPER_CONSTRAINTS_FILE -r requirements.txt -r test-requirements.txt &&
pip install .)
}
# Extract messages for a django project, we need to update django.pot
# and djangojs.pot.
function extract_messages_django {
local modulename=$1
local pot
KEYWORDS="-k gettext_noop -k gettext_lazy -k ngettext_lazy:1,2"
KEYWORDS+=" -k ugettext_noop -k ugettext_lazy -k ungettext_lazy:1,2"
KEYWORDS+=" -k npgettext:1c,2,3 -k pgettext_lazy:1c,2 -k npgettext_lazy:1c,2,3"
for DOMAIN in djangojs django ; do
if [ -f babel-${DOMAIN}.cfg ]; then
mkdir -p ${modulename}/locale
pot=${modulename}/locale/${DOMAIN}.pot
touch ${pot}
pybabel ${QUIET} extract -F babel-${DOMAIN}.cfg \
--add-comments Translators: \
--msgid-bugs-address="https://bugs.launchpad.net/openstack-i18n/" \
--project=${PROJECT} --version=${VERSION} \
$KEYWORDS \
-o ${pot} ${modulename}
check_empty_pot ${pot}
fi
done
}
# Extract doc messages
function extract_messages_doc {
# Temporary build folder for gettext
mkdir -p doc/build/gettext
# Extract messages
sphinx-build -b gettext doc/source \
doc/build/gettext/
# Manipulates pot translation sources if needed
if [[ -f tools/doc-pot-filter.sh ]]; then
tools/doc-pot-filter.sh
fi
# New translation target projects may not have locale folder
mkdir -p doc/source/locale
# Sphinx builds a pot file for each directory and for each file
# in the top-level directory.
# We keep the directory files and concatenate all top-level files.
local has_other=0
for f in doc/build/gettext/*.pot; do
local fn=$(basename $f .pot)
# If a pot file corresponds to a directory, we use the pot file as-is.
if [ -d doc/source/$fn ]; then
# Remove UUIDs, those are not necessary and change too often
msgcat --use-first --sort-by-file $f | \
awk '$0 !~ /^\# [a-z0-9]+$/' > doc/source/locale/doc-$fn.pot
rm $f
else
has_other=1
fi
done
# We concatenate remaining into a single pot file so that
# "git add ${DIRECTORY}/source/locale" will only add a
# single pot file for all top-level files.
if [ "$has_other" = "1" ]; then
# Remove UUIDs, those are not necessary and change too often
msgcat --use-first --sort-by-file doc/build/gettext/*.pot | \
awk '$0 !~ /^\# [a-z0-9]+$/' > doc/source/locale/doc.pot
fi
rm -rf doc/build/gettext/
}
# Extract releasenotes messages
function extract_messages_releasenotes {
local keep_workdir=$1
# Extract messages
$HOME/.venv/bin/sphinx-build -b gettext -d releasenotes/build/doctrees \
releasenotes/source releasenotes/work
rm -rf releasenotes/build
# Concatenate messages into one POT file
mkdir -p releasenotes/source/locale/
msgcat --sort-by-file releasenotes/work/*.pot \
> releasenotes/source/locale/releasenotes.pot
if [ ! -n "$keep_workdir" ]; then
rm -rf releasenotes/work
fi
}
# Check releasenote translation progress per language.
# It checks the progress per release. Add the release note translation
# if at least one release is well translated (>= 75%).
# Keep the release note translation in the git repository
# if at least one release is translated >= 40%.
# Otherwise (< 40%) the translation are removed.
#
# NOTE: this function assume POT files in releasenotes/work
# extracted by extract_messages_releasenotes().
# The workdir should be clean up by the caller.
function check_releasenotes_per_language {
local lang_po=$1
# The expected PO location is
# releasenotes/source/locale/<lang>/LC_MESSAGES/releasenotes.po.
# Extract language name from 4th component.
local lang
lang=$(echo $lang_po | cut -d / -f 4)
local release_pot
local release_name
local workdir=releasenotes/work
local has_high_thresh=0
local has_low_thresh=0
mkdir -p $workdir/$lang
for release_pot in $(find $workdir -name '*.pot'); do
release_name=$(basename $release_pot .pot)
# The index file usually contains small number of words,
# so we skip to check it.
if [ $release_name = "index" ]; then
continue
fi
msgmerge --quiet -o $workdir/$lang/$release_name.po $lang_po $release_pot
check_po_file $workdir/$lang/$release_name.po
if [ $RATIO -ge 75 ]; then
has_high_thresh=1
has_low_thresh=1
fi
if [ $RATIO -ge 40 ]; then
has_low_thresh=1
fi
done
if ! git ls-files | grep -xq $lang_po; then
if [ $has_high_thresh -eq 0 ]; then
rm -f $lang_po
fi
else
if [ $has_low_thresh -eq 0 ]; then
git rm -f --ignore-unmatch $lang_po
fi
fi
}
# Filter out files that we do not want to commit.
# Sets global variable INVALID_PO_FILE to 1 if any invalid files are
# found.
function filter_commits {
local ret
# Don't add new empty files.
for f in $(git diff --cached --name-only --diff-filter=A); do
case "$f" in
*.po)
# Files should have at least one non-empty msgid string.
if ! grep -q 'msgid "[^"]' "$f" ; then
git reset -q "$f"
rm "$f"
fi
;;
*.json)
# JSON files fail msgid test. Ignore the locale key and confirm
# there are string keys in the messages dictionary itself.
if ! grep -q '"[^"].*":\s*"' "$f" ; then
git reset -q "$f"
rm "$f"
fi
;;
*)
# Anything else is not a translation file, remove it.
git reset -q "$f"
rm "$f"
;;
esac
done
# Don't send files where the only things which have changed are
# the creation date, the version number, the revision date,
# name of last translator, comment lines, or diff file information.
REAL_CHANGE=0
# Always remove obsolete log level translations.
for f in $(git diff --cached --name-only --diff-filter=D); do
if [[ $f =~ -log-(critical|error|info|warning).po$ ]] ; then
REAL_CHANGE=1
fi
done
# Don't iterate over deleted files
for f in $(git diff --cached --name-only --diff-filter=AM); do
# It's ok if the grep fails
set +e
REGEX="(POT-Creation-Date|Project-Id-Version|PO-Revision-Date|Last-Translator|X-Generator|Generated-By)"
changed=$(git diff --cached "$f" \
| egrep -v "$REGEX" \
| egrep -c "^([-+][^-+#])")
added=$(git diff --cached "$f" \
| egrep -v "$REGEX" \
| egrep -c "^([+][^+#])")
set -e
# Check that imported po files are valid
if [[ $f =~ .po$ ]] ; then
set +e
msgfmt --check-format -o /dev/null $f
ret=$?
set -e
if [ $ret -ne 0 ] ; then
# Set change to zero so that next expression reverts
# change of this file.
changed=0
echo "ERROR: File $f is an invalid po file."
echo "ERROR: The file has not been imported and needs fixing!"
INVALID_PO_FILE=1
fi
fi
if [ $changed -eq 0 ]; then
git reset -q "$f"
git checkout -- "$f"
# We will take this import if at least one change adds new content,
# thus adding a new translation.
# If only lines are removed, we do not need to generate an import.
elif [ $added -gt 0 ] ; then
REAL_CHANGE=1
fi
done
# If no file has any real change, revert all changes.
if [ $REAL_CHANGE -eq 0 ] ; then
# New files need to be handled differently
for f in $(git diff --cached --name-only --diff-filter=A) ; do
git reset -q -- "$f"
rm "$f"
done
for f in $(git diff --cached --name-only) ; do
git reset -q -- "$f"
git checkout -- "$f"
done
fi
}
# Check the amount of translation done for a .po file, sets global variable
# RATIO.
function check_po_file {
local file=$1
local dropped_ratio=$2
trans=$(msgfmt --statistics -o /dev/null "$file" 2>&1)
check="^0 translated messages"
if [[ $trans =~ $check ]] ; then
RATIO=0
else
if [[ $trans =~ " translated message" ]] ; then
trans_no=$(echo $trans|sed -e 's/ translated message.*$//')
else
trans_no=0
fi
if [[ $trans =~ " untranslated message" ]] ; then
untrans_no=$(echo $trans|sed -e 's/^.* \([0-9]*\) untranslated message.*/\1/')
else
untrans_no=0
fi
total=$(($trans_no+$untrans_no))
RATIO=$((100*$trans_no/$total))
fi
}
# Remove obsolete files. We might have added them in the past but
# would not add them today, so let's eventually remove them.
function cleanup_po_files {
local modulename=$1
for i in $(find $modulename -name *.po) ; do
check_po_file "$i"
if [ $RATIO -lt 40 ]; then
git rm -f --ignore-unmatch $i
fi
done
}
# Remove obsolete log lovel files. We have added them in the past but
# do not translate them anymore, so let's eventually remove them.
function cleanup_log_files {
local modulename=$1
local levels="info warning error critical"
for i in $(find $modulename -name *.po) ; do
# We do not store the log level files anymore, remove them
# from git.
local bi
bi=$(basename $i)
for level in $levels ; do
if [[ "$bi" == "$modulename-log-$level.po" ]] ; then
git rm -f --ignore-unmatch $i
fi
done
done
}
# Remove all pot files, we publish them to
# http://tarballs.openstack.org/translation-source/{name}/VERSION ,
# let's not store them in git at all.
# Previously, we had those files in git, remove them now if there
# are still there.
function cleanup_pot_files {
local modulename=$1
for i in $(find $modulename -name *.pot) ; do
# Remove file; both local and from git if needed.
rm $i
git rm -f --ignore-unmatch $i
done
}
# Reduce size of po files. This reduces the amount of content imported
# and makes for fewer imports.
# This does not touch the pot files. This way we can reconstruct the po files
# using "msgmerge POTFILE POFILE -o COMPLETEPOFILE".
function compress_po_files {
local directory=$1
for i in $(find $directory -name *.po) ; do
# Remove auto translated comments, those are not needed.
# Do this first since it introduces empty lines, the msgattrib
# remove them.
sed -e 's|^# auto translated by TM merge.*$||' -i $i
msgattrib --translated --no-location --sort-output "$i" \
--output="${i}.tmp"
mv "${i}.tmp" "$i"
done
}
function pull_from_zanata {
local project=$1
# Since Zanata does not currently have an option to not download new
# files, we download everything, and then remove new files that are not
# translated enough.
zanata-cli -B -e pull
# We skip directories starting with '.' because they never contain
# translations for the project (in particular, '.tox'). Likewise
# 'node_modules' only contains dependencies and should be ignored.
for i in $(find . -name '*.po' ! -path './.*' ! -path './node_modules/*' -prune | cut -b3-); do
# We check release note translation percentage per release.
# To check this we need to extract messages per RST file.
# Let's defer checking it to propose_releasenotes.
local basefn=
if [ "$(basename $i)" = "releasenotes.po" ]; then
continue
fi
check_po_file "$i"
# We want new files to be >75% translated. The
# common documents in openstack-manuals have that relaxed to
# >40%.
percentage=75
if [ $project = "openstack-manuals" ]; then
case "$i" in
*common*)
percentage=40
;;
esac
fi
if [ $RATIO -lt $percentage ]; then
# This means the file is below the ratio, but we only want
# to delete it, if it is a new file. Files known to git
# that drop below 40% will be cleaned up by
# cleanup_po_files.
if ! git ls-files | grep -xq "$i"; then
rm -f "$i"
fi
fi
done
}
# Copy all pot files in modulename directory to temporary path for
# publishing. This uses the exact same path.
function copy_pot {
local all_modules=$1
local target=.translation-source/$ZANATA_VERSION/
for m in $all_modules ; do
for f in `find $m -name "*.pot" ` ; do
local fd
fd=$(dirname $f)
mkdir -p $target/$fd
cp $f $target/$f
done
done
}