Translations: Fix releasenotes

Since we have now first translations of releasenotes, we can test
downloading of these. Testing barbican, nova, and openstack-manuals on
my local system found a couple of problems that are fixed with this
patch.

Refactor how we handle releasenotes, let's handle them like python and
django projects with specific functions.

Run the same cleanup on releasenotes like on other pot and po
files.

Fix download of releasenotes translations for openstack-manuals, previously
zanata-cli complained about a missing rule for these.

Remove from setup_project the special reno_resource setup, this is not
needed, the general rule works just fine.

Also, openstack-manuals does not set up the venv, so no need to remove
it.

Include minor cleanup for cleanup_pot_files.

Change-Id: I8c9a018b859b983f39c5239224bb530ce7500cb3

jenkins/scripts/common_translation_update.sh

@@ -62,7 +62,7 @@ function setup_project {
     shift 2
     # All argument(s) contain module names now.
 
-    local EXCLUDE='.tox/**'
+    local exclude='.tox/**'
 
     # For projects with one module on stable/mitaka, we use "old" setup.
     # Note that stable/mitaka is only stable translated branch for
@@ -76,17 +76,10 @@ function setup_project {
             -r '**/*.pot' '{locale_with_underscore}/LC_MESSAGES/{filename}.po' \
             -f zanata.xml
     else
-        local reno_resource=""
-
-        if [ "$version" == "master" ] && [ -f releasenotes/source/conf.py ]; then
-            translate_reno=1
-            extract_messages_releasenotes
-            reno_resource="-r releasenotes/source/locale/releasenotes.pot releasenotes/source/locale/{locale_with_underscore}/LC_MESSAGES/releasenotes.po"
-        fi
         /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
             -p $project -v $version --srcdir . --txdir . \
             -r '**/*.pot' '{path}/{locale_with_underscore}/LC_MESSAGES/{filename}.po' \
-            $reno_resource -e "$EXCLUDE" -f zanata.xml
+            -e "$exclude" -f zanata.xml
     fi
 }
 
@@ -166,14 +159,13 @@ function setup_manuals {
     done
 
     # Project setup and updating POT files for release notes.
-    # As first step, execute for openstack-manuals only.
     if [[ $project == "openstack-manuals" ]] && [[ $version == "master" ]]; then
-        extract_messages_releasenotes
+        ZANATA_RULES="$ZANATA_RULES -r ./releasenotes/source/locale/releasenotes.pot releasenotes/source/locale/{locale_with_underscore}/LC_MESSAGES/releasenotes.po"
-        ZANATA_RULES="$ZANATA_RULES -r releasenotes/source/locale/releasenotes.pot releasenotes/source/locale/{locale_with_underscore}/LC_MESSAGES/releasenotes.po"
     fi
 
-    /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project \
+    /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
-        -v $version --srcdir . --txdir . $ZANATA_RULES -e "$EXCLUDE" \
+        -p $project -v $version --srcdir . --txdir . \
+        $ZANATA_RULES -e "$EXCLUDE" \
         -f zanata.xml
 }
 
@@ -185,8 +177,9 @@ function setup_training_guides {
     # Update the .pot file
     tox -e generatepot-training
 
-    /usr/local/jenkins/slave_scripts/create-zanata-xml.py -p $project \
+    /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
-        -v $version --srcdir doc/upstream-training/source/locale \
+        -p $project -v $version \
+        --srcdir doc/upstream-training/source/locale \
         --txdir doc/upstream-training/source/locale \
         -f zanata.xml
 }
@@ -378,7 +371,6 @@ function extract_messages_releasenotes {
     msgcat --sort-by-file releasenotes/work/*.pot \
         > releasenotes/source/locale/releasenotes.pot
     rm -rf releasenotes/work
-    git add releasenotes/source/locale/releasenotes.pot
 }
 
 # Filter out files that we do not want to commit.
@@ -486,7 +478,7 @@ function check_po_file {
 function cleanup_po_files {
     local modulename=$1
 
-    for i in $(find $modulename/locale -name *.po) ; do
+    for i in $(find $modulename -name *.po) ; do
         check_po_file "$i"
         if [ $RATIO -lt 20 ]; then
             git rm -f --ignore-unmatch $i
@@ -500,14 +492,15 @@ function cleanup_po_files {
 function cleanup_pot_files {
     local modulename=$1
 
-    for i in $(find $modulename/locale -name *.pot) ; do
+    for i in $(find $modulename -name *.pot) ; do
         local bi
         local bi_po
         local count_po
 
-        bi=$(basename $i)
+        # Get basename and remove .pot suffix from file name
-        bi_po="${bi%.pot}.po"
+        bi=$(basename $i .pot)
-        count_po=$(find $modulename/locale -name "${bi_po}"|wc -l)
+        bi_po="${bi}.po"
+        count_po=$(find $modulename -name "${bi_po}" | wc -l)
         if [ $count_po -eq 0 ] ; then
             # Remove file, it might be a new file unknown to git.
             rm $i
@@ -560,7 +553,6 @@ function pull_from_zanata {
     # translated enough.
     zanata-cli -B -e pull
 
-
     for i in $(find . -name '*.po' ! -path './.*' -prune | cut -b3-); do
         check_po_file "$i"
         # We want new files to be >75% translated. The glossary and

jenkins/scripts/propose_translation_update.sh

@@ -113,6 +113,33 @@ function propose_django {
     git add $modulename/locale/
 }
 
+
+function propose_releasenotes {
+    local version=$1
+
+    # This function does not check whether releasenote publishing and
+    # testing are set up in zuul/layout.yaml. If releasenotes exist,
+    # they get pushed to the translation server.
+
+    if [[ "$version" == "master" && -f releasenotes/source/conf.py ]]; then
+        extract_messages_releasenotes
+
+        # Remove obsolete files.
+        cleanup_po_files "releasenotes"
+        cleanup_pot_files "releasenotes"
+
+        # Compress downloaded po files
+        compress_po_files "releasenotes"
+
+        # Add all changed files to git - if there are
+        # translated files at all.
+        if [ -d releasenotes/source/locale/ ] ; then
+            git add releasenotes/source/locale/
+        fi
+    fi
+}
+
+
 # Setup git repository for git review.
 setup_git
 
@@ -124,6 +151,7 @@ case "$PROJECT" in
         init_manuals "$PROJECT"
         setup_manuals "$PROJECT" "$ZANATA_VERSION"
         propose_manuals
+        propose_releasenotes "$ZANATA_VERSION"
         ;;
     training-guides)
         setup_training_guides "$ZANATA_VERSION"
@@ -139,6 +167,7 @@ case "$PROJECT" in
             setup_loglevel_vars
             # Pull updated translations from Zanata
             pull_from_zanata "$PROJECT"
+            propose_releasenotes "$ZANATA_VERSION"
             for modulename in $module_names; do
                 propose_python "$modulename"
             done
@@ -151,6 +180,7 @@ case "$PROJECT" in
             install_horizon
             # Pull updated translations from Zanata.
             pull_from_zanata "$PROJECT"
+            propose_releasenotes "$ZANATA_VERSION"
             for modulename in $module_names; do
                 propose_django "$modulename"
             done

jenkins/scripts/upstream_translation_update.sh

@@ -36,6 +36,9 @@ case "$PROJECT" in
         init_manuals "$PROJECT"
         # POT file extraction is done in setup_manuals.
         setup_manuals "$PROJECT" "$ZANATA_VERSION"
+        if [[ "$ZANATA_VERSION" == "master" && -f releasenotes/source/conf.py ]]; then
+            extract_messages_releasenotes
+        fi
         ;;
     training-guides)
         setup_training_guides "$ZANATA_VERSION"
@@ -48,6 +51,7 @@ case "$PROJECT" in
         if [ -n "$module_names" ]; then
             setup_project "$PROJECT" "$ZANATA_VERSION" $module_names
             setup_loglevel_vars
+            extract_messages_releasenotes
             for modulename in $module_names; do
                 extract_messages "$modulename"
                 extract_messages_log "$modulename"
@@ -59,6 +63,7 @@ case "$PROJECT" in
         if [ -n "$module_names" ]; then
             setup_project "$PROJECT" "$ZANATA_VERSION" $module_names
             install_horizon
+            extract_messages_releasenotes
             for modulename in $module_names; do
                 extract_messages_django "$modulename"
             done
@@ -72,11 +77,16 @@ esac
 if [[ ! $PROJECT =~ api-site|ha-guide|openstack-manuals|operations-guide|security-doc ]]; then
     git add */locale/*
 fi
+if [ -d releasenotes/source/locale ] ; then
+    git add releasenotes/source/locale
+fi
 
 if [ $(git diff --cached | egrep -v "(POT-Creation-Date|^[\+\-]#|^\+{3}|^\-{3})" | egrep -c "^[\-\+]") -gt 0 ]; then
     # First, delete our VENV, otherwise the Zanata client might push some
     # extra files.
-    rm -rf $VENV
+    if [ "$VENV" != "" ] ; then
+        rm -rf $VENV
+    fi
     # The Zanata client works out what to send based on the zanata.xml file.
     # Do not copy translations from other files for this change.
     zanata-cli -B -e push --copy-trans False