From 5cb5247c5c1133e999b065aa6ff8d4a6991d23ac Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Sun, 17 Apr 2016 19:21:12 +0200 Subject: [PATCH] Translations: Fix releasenotes Since we have now first translations of releasenotes, we can test downloading of these. Testing barbican, nova, and openstack-manuals on my local system found a couple of problems that are fixed with this patch. Refactor how we handle releasenotes, let's handle them like python and django projects with specific functions. Run the same cleanup on releasenotes like on other pot and po files. Fix download of releasenotes translations for openstack-manuals, previously zanata-cli complained about a missing rule for these. Remove from setup_project the special reno_resource setup, this is not needed, the general rule works just fine. Also, openstack-manuals does not set up the venv, so no need to remove it. Include minor cleanup for cleanup_pot_files. Change-Id: I8c9a018b859b983f39c5239224bb530ce7500cb3 --- jenkins/scripts/common_translation_update.sh | 38 ++++++++----------- jenkins/scripts/propose_translation_update.sh | 30 +++++++++++++++ .../scripts/upstream_translation_update.sh | 12 +++++- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/jenkins/scripts/common_translation_update.sh b/jenkins/scripts/common_translation_update.sh index 1dfb6672c7..5fb657027c 100644 --- a/jenkins/scripts/common_translation_update.sh +++ b/jenkins/scripts/common_translation_update.sh @@ -62,7 +62,7 @@ function setup_project { shift 2 # All argument(s) contain module names now. - local EXCLUDE='.tox/**' + local exclude='.tox/**' # For projects with one module on stable/mitaka, we use "old" setup. # Note that stable/mitaka is only stable translated branch for @@ -76,17 +76,10 @@ function setup_project { -r '**/*.pot' '{locale_with_underscore}/LC_MESSAGES/{filename}.po' \ -f zanata.xml else - local reno_resource="" - - if [ "$version" == "master" ] && [ -f releasenotes/source/conf.py ]; then - translate_reno=1 - extract_messages_releasenotes - reno_resource="-r releasenotes/source/locale/releasenotes.pot releasenotes/source/locale/{locale_with_underscore}/LC_MESSAGES/releasenotes.po" - fi /usr/local/jenkins/slave_scripts/create-zanata-xml.py \ -p $project -v $version --srcdir . --txdir . \ -r '**/*.pot' '{path}/{locale_with_underscore}/LC_MESSAGES/{filename}.po' \ - $reno_resource -e "$EXCLUDE" -f zanata.xml + -e "$exclude" -f zanata.xml fi } @@ -166,14 +159,13 @@ function setup_manuals { done # Project setup and updating POT files for release notes. - # As first step, execute for openstack-manuals only. if [[ $project == "openstack-manuals" ]] && [[ $version == "master" ]]; then - extract_messages_releasenotes - ZANATA_RULES="$ZANATA_RULES -r releasenotes/source/locale/releasenotes.pot releasenotes/source/locale/{locale_with_underscore}/LC_MESSAGES/releasenotes.po" + ZANATA_RULES="$ZANATA_RULES -r ./releasenotes/source/locale/releasenotes.pot releasenotes/source/locale/{locale_with_underscore}/LC_MESSAGES/releasenotes.po" fi - /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project \ - -v $version --srcdir . --txdir . $ZANATA_RULES -e "$EXCLUDE" \ + /usr/local/jenkins/slave_scripts/create-zanata-xml.py \ + -p $project -v $version --srcdir . --txdir . \ + $ZANATA_RULES -e "$EXCLUDE" \ -f zanata.xml } @@ -185,8 +177,9 @@ function setup_training_guides { # Update the .pot file tox -e generatepot-training - /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project \ - -v $version --srcdir doc/upstream-training/source/locale \ + /usr/local/jenkins/slave_scripts/create-zanata-xml.py \ + -p $project -v $version \ + --srcdir doc/upstream-training/source/locale \ --txdir doc/upstream-training/source/locale \ -f zanata.xml } @@ -378,7 +371,6 @@ function extract_messages_releasenotes { msgcat --sort-by-file releasenotes/work/*.pot \ > releasenotes/source/locale/releasenotes.pot rm -rf releasenotes/work - git add releasenotes/source/locale/releasenotes.pot } # Filter out files that we do not want to commit. @@ -486,7 +478,7 @@ function check_po_file { function cleanup_po_files { local modulename=$1 - for i in $(find $modulename/locale -name *.po) ; do + for i in $(find $modulename -name *.po) ; do check_po_file "$i" if [ $RATIO -lt 20 ]; then git rm -f --ignore-unmatch $i @@ -500,14 +492,15 @@ function cleanup_po_files { function cleanup_pot_files { local modulename=$1 - for i in $(find $modulename/locale -name *.pot) ; do + for i in $(find $modulename -name *.pot) ; do local bi local bi_po local count_po - bi=$(basename $i) - bi_po="${bi%.pot}.po" - count_po=$(find $modulename/locale -name "${bi_po}"|wc -l) + # Get basename and remove .pot suffix from file name + bi=$(basename $i .pot) + bi_po="${bi}.po" + count_po=$(find $modulename -name "${bi_po}" | wc -l) if [ $count_po -eq 0 ] ; then # Remove file, it might be a new file unknown to git. rm $i @@ -560,7 +553,6 @@ function pull_from_zanata { # translated enough. zanata-cli -B -e pull - for i in $(find . -name '*.po' ! -path './.*' -prune | cut -b3-); do check_po_file "$i" # We want new files to be >75% translated. The glossary and diff --git a/jenkins/scripts/propose_translation_update.sh b/jenkins/scripts/propose_translation_update.sh index 175cc86f23..f4fb6ef289 100755 --- a/jenkins/scripts/propose_translation_update.sh +++ b/jenkins/scripts/propose_translation_update.sh @@ -113,6 +113,33 @@ function propose_django { git add $modulename/locale/ } + +function propose_releasenotes { + local version=$1 + + # This function does not check whether releasenote publishing and + # testing are set up in zuul/layout.yaml. If releasenotes exist, + # they get pushed to the translation server. + + if [[ "$version" == "master" && -f releasenotes/source/conf.py ]]; then + extract_messages_releasenotes + + # Remove obsolete files. + cleanup_po_files "releasenotes" + cleanup_pot_files "releasenotes" + + # Compress downloaded po files + compress_po_files "releasenotes" + + # Add all changed files to git - if there are + # translated files at all. + if [ -d releasenotes/source/locale/ ] ; then + git add releasenotes/source/locale/ + fi + fi +} + + # Setup git repository for git review. setup_git @@ -124,6 +151,7 @@ case "$PROJECT" in init_manuals "$PROJECT" setup_manuals "$PROJECT" "$ZANATA_VERSION" propose_manuals + propose_releasenotes "$ZANATA_VERSION" ;; training-guides) setup_training_guides "$ZANATA_VERSION" @@ -139,6 +167,7 @@ case "$PROJECT" in setup_loglevel_vars # Pull updated translations from Zanata pull_from_zanata "$PROJECT" + propose_releasenotes "$ZANATA_VERSION" for modulename in $module_names; do propose_python "$modulename" done @@ -151,6 +180,7 @@ case "$PROJECT" in install_horizon # Pull updated translations from Zanata. pull_from_zanata "$PROJECT" + propose_releasenotes "$ZANATA_VERSION" for modulename in $module_names; do propose_django "$modulename" done diff --git a/jenkins/scripts/upstream_translation_update.sh b/jenkins/scripts/upstream_translation_update.sh index 94426fead3..6e0e8aa4c5 100755 --- a/jenkins/scripts/upstream_translation_update.sh +++ b/jenkins/scripts/upstream_translation_update.sh @@ -36,6 +36,9 @@ case "$PROJECT" in init_manuals "$PROJECT" # POT file extraction is done in setup_manuals. setup_manuals "$PROJECT" "$ZANATA_VERSION" + if [[ "$ZANATA_VERSION" == "master" && -f releasenotes/source/conf.py ]]; then + extract_messages_releasenotes + fi ;; training-guides) setup_training_guides "$ZANATA_VERSION" @@ -48,6 +51,7 @@ case "$PROJECT" in if [ -n "$module_names" ]; then setup_project "$PROJECT" "$ZANATA_VERSION" $module_names setup_loglevel_vars + extract_messages_releasenotes for modulename in $module_names; do extract_messages "$modulename" extract_messages_log "$modulename" @@ -59,6 +63,7 @@ case "$PROJECT" in if [ -n "$module_names" ]; then setup_project "$PROJECT" "$ZANATA_VERSION" $module_names install_horizon + extract_messages_releasenotes for modulename in $module_names; do extract_messages_django "$modulename" done @@ -72,11 +77,16 @@ esac if [[ ! $PROJECT =~ api-site|ha-guide|openstack-manuals|operations-guide|security-doc ]]; then git add */locale/* fi +if [ -d releasenotes/source/locale ] ; then + git add releasenotes/source/locale +fi if [ $(git diff --cached | egrep -v "(POT-Creation-Date|^[\+\-]#|^\+{3}|^\-{3})" | egrep -c "^[\-\+]") -gt 0 ]; then # First, delete our VENV, otherwise the Zanata client might push some # extra files. - rm -rf $VENV + if [ "$VENV" != "" ] ; then + rm -rf $VENV + fi # The Zanata client works out what to send based on the zanata.xml file. # Do not copy translations from other files for this change. zanata-cli -B -e push --copy-trans False