Redo function that pulls translations from Zanata

The existing function pull_from_zanata would only work for the simple
Python project case of pulling translations from Zanata -- with only
locales directory and trivial rules to apply, with no real hope of
working for Horizon or OpenStack Manuals, given the current
functionality of the Zanata CLI utility.

Rewrite it to download every translation from Zanata and perform some
checks by hand if we keep the file or remove it. This also required
refactoring a large portion of cleanup_po_files so both functions could
leverage the same code.  OpenStack Manuals treats downloaded
translations sufficiently different that a similar function was required.

This also adds full support for Horizon and OpenStack Manuals projects,
including jobs, as well as new job templates to make it easier to
transition projects to pulling from Zanata. Adding -zanata jobs for the
Python projects and django_openstack_auth is not required, they are
already present.

Change-Id: I5b9f2eca7f099c56057fc5d09b4bbfe5ea5c596d

jenkins/jobs/translation-jobs.yaml

@@ -131,6 +131,23 @@
 
     node: 'proposal'
 
+- job:
+    name: 'horizon-propose-translation-update-zanata'
+
+    builders:
+      - revoke-sudo
+      - branch-git-prep:
+          branch: master
+      - shell: |
+          #!/bin/bash -xe
+          /usr/local/jenkins/slave_scripts/propose_translation_update_horizon.sh zanata
+
+    publishers:
+      - console-log
+      - proposal-slave-cleanup
+
+    node: 'proposal'
+
 - job-template:
     name: '{name}-manuals-upstream-translation-update'
 
@@ -164,12 +181,35 @@
 
     node: 'proposal'
 
+- job-template:
+    name: '{name}-manuals-propose-translation-update-zanata'
+
+    builders:
+      - revoke-sudo
+      - branch-git-prep:
+          branch: master
+      - shell: |
+          #!/bin/bash -xe
+          /usr/local/jenkins/slave_scripts/propose_translation_update_manuals.sh {name} zanata
+
+    publishers:
+      - console-log
+      - proposal-slave-cleanup
+
+    node: 'proposal'
+
 - job-group:
     name: translation-jobs
     jobs:
       - '{name}-upstream-translation-update'
       - '{name}-propose-translation-update'
 
+- job-group:
+    name: translation-jobs-zanata
+    jobs:
+      - '{name}-upstream-translation-update'
+      - '{name}-propose-translation-update-zanata'
+
 - job-group:
     name: manual-translation-jobs
     jobs:
@@ -179,3 +219,13 @@
           envlist: checklang
       - '{name}-tox-doc-{envlist}':
           envlist: publishlang
+
+- job-group:
+    name: manual-translation-jobs-zanata
+    jobs:
+      - '{name}-manuals-upstream-translation-update'
+      - '{name}-manuals-propose-translation-update-zanata'
+      - 'gate-{name}-tox-{envlist}':
+          envlist: checklang
+      - '{name}-tox-doc-{envlist}':
+          envlist: publishlang

jenkins/scripts/common_translation_update.sh

@@ -389,39 +389,41 @@ function filter_commits {
     fi
 }
 
+# Check the amount of translation done for a .po file, sets global variable
+# RATIO.
+function check_po_file {
+    local file=$1
+    local dropped_ratio=$2
+
+    trans=$(msgfmt --statistics -o /dev/null "$file" 2>&1)
+    check="^0 translated messages"
+    if [[ $trans =~ $check ]] ; then
+        RATIO=0
+    else
+        if [[ $trans =~ " translated message" ]] ; then
+            trans_no=$(echo $trans|sed -e 's/ translated message.*$//')
+        else
+            trans_no=0
+        fi
+        if [[ $trans =~ " untranslated message" ]] ; then
+            untrans_no=$(echo $trans|sed -e 's/^.* \([0-9]*\) untranslated message.*/\1/')
+        else
+            untrans_no=0
+        fi
+        total=$(($trans_no+$untrans_no))
+        RATIO=$((100*$trans_no/$total))
+    fi
+}
+
 # Remove obsolete files. We might have added them in the past but
 # would not add them today, so let's eventually remove them.
 function cleanup_po_files {
     local project=$1
 
     for i in $(find $project/locale -name *.po) ; do
-        # Output goes to stderr, so redirect to stdout to catch it.
-        trans=$(msgfmt --statistics -o /dev/null "$i" 2>&1)
-        check="^0 translated messages"
-        if [[ $trans =~ $check ]] ; then
-            # Nothing is translated, remove the file.
-            git rm -f "$i"
-        else
-            if [[ $trans =~ " translated message" ]] ; then
-                trans_no=$(echo $trans|sed -e 's/ translated message.*$//')
-            else
-                trans_no=0
-            fi
-            if [[ $trans =~ " untranslated message" ]] ; then
-                untrans_no=$(echo $trans|sed -e 's/^.* \([0-9]*\) untranslated message.*/\1/')
-            else
-                untrans_no=0
-            fi
-            let total=$trans_no+$untrans_no
-            let ratio=100*$trans_no/$total
-            # Since we only download files that are at least
-            # translated to 75 per cent, let's delete those that have
-            # signficantly less translations.
-            # For now we delete files that suddenly are less than 20
-            # per cent translated.
-            if [[ "$ratio" -lt "20" ]] ; then
-                git rm -f "$i"
-            fi
+        check_po_file "$i"
+        if [ $RATIO -lt 20 ]; then
+            git rm -f $i
         fi
     done
 }
@@ -473,16 +475,53 @@ function pull_from_transifex {
 }
 
 function pull_from_zanata {
-    local base_dir=$1
+    # Since Zanata does not currently have an option to not download new
+    # files, we download everything, and then remove new files that are not
+    # translated enough.
+    zanata-cli -B -e pull
 
-    # Download all files that are at least 75% translated.
-    zanata-cli -B -e pull --min-doc-percent 75
-
-    # Work out existing locales, and only pull them. This will download
-    # updates for existing translations that don't meet the 75% translated
-    # criterion.
-    locales=$(ls $base_dir/locale | grep -v pot | tr '\n' ',' | sed 's/,$//')
-    if [ -n "$locales" ]; then
-        zanata-cli -B -e pull -l $locales
-    fi
+    for i in $(find . -name '*.po' ! -path './.*' -prune | cut -b3-); do
+        check_po_file "$i"
+        if [ $RATIO -lt 75 ]; then
+            # This means the file is below the ratio, but we only want to
+            # delete it if is a new file. Files known to git that drop below
+            # 20% will be cleaned up by cleanup_po_files.
+            if ! git ls-files | grep -xq "$i"; then
+                rm -f "$i"
+            fi
+        fi
+    done
+}
+
+function pull_from_zanata_manuals {
+    local project=$1
+
+    # Since Zanata does not currently have an option to not download new
+    # files, we download everything, and then remove new files that are not
+    # translated enough.
+    zanata-cli -B -e pull
+
+    for i in $(find . -name '*.po' ! -path './.*' -prune | cut -b3-); do
+        # We want new files to be >75% translated. The glossary and common
+        # documents in openstack-manuals have that relaxed to >8%.
+        percentage=75
+        if [ $project = "openstack-manuals" ]; then
+            case "$i" in
+                *glossary*|*common*)
+                    percentage=8
+                    ;;
+            esac
+        fi
+        check_po_file "$i"
+        if git ls-files | grep -xq "$i"; then
+            # Existing file, we only want to update it if it's >50% translated.
+            if [ $RATIO -lt 50 ]; then
+                git checkout "$i"
+            fi
+        else
+            if [ $RATIO -lt $percentage ]; then
+                rm -f "$i"
+            fi
+        fi
+    done
 }

jenkins/scripts/propose_translation_update.sh

@@ -42,7 +42,7 @@ case "$SOFTWARE" in
         pull_from_transifex
         ;;
     Zanata)
-        pull_from_zanata "$PROJECT"
+        pull_from_zanata
         ;;
 esac
 

jenkins/scripts/propose_translation_update_django_openstack_auth.sh

@@ -35,7 +35,7 @@ case "$SOFTWARE" in
         pull_from_transifex
         ;;
     Zanata)
-        pull_from_zanata "openstack_auth"
+        pull_from_zanata
         ;;
 esac
 

jenkins/scripts/propose_translation_update_horizon.sh

@@ -12,16 +12,29 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+SOFTWARE="Transifex"
+
+if [ -n "$1" -a "$1" = "zanata" ]; then
+    SOFTWARE="Zanata"
+fi
+
 source /usr/local/jenkins/slave_scripts/common_translation_update.sh
 
 setup_git
 
-setup_review
+setup_review "$SOFTWARE"
 setup_translation
 setup_horizon
 
-# Pull updated translations from Transifex.
-pull_from_transifex
+# Pull updated translations from Transifex, or Zanata.
+case "$SOFTWARE" in
+    Transifex)
+        pull_from_transifex
+        ;;
+    Zanata)
+        pull_from_zanata
+        ;;
+esac
 
 # Invoke run_tests.sh to update the po files
 # Or else, "../manage.py makemessages" can be used.

jenkins/scripts/propose_translation_update_manuals.sh

@@ -18,28 +18,41 @@
 # and push to Gerrit.
 
 PROJECT=$1
+SOFTWARE="Transifex"
+
+if [ -n "$2" -a "$2" = "zanata" ]; then
+    SOFTWARE="Zanata"
+fi
 
 source /usr/local/jenkins/slave_scripts/common_translation_update.sh
 
 init_manuals "$PROJECT"
 
 setup_git
-setup_review
+setup_review "$SOFTWARE"
 setup_translation
 
 setup_manuals "$PROJECT"
 
-# Download new files.
-# Also downloads updates for existing files that are
-# translated to a certain amount as configured in setup_manuals.
-# The function setup_manuals will setup most files --minimum-perc=75
-# for most files.
-tx pull -a -f
+# Pull updated translations from Transifex, or Zanata.
+case "$SOFTWARE" in
+    Transifex)
+        # Download new files.
+        # Also downloads updates for existing files that are
+        # translated to a certain amount as configured in setup_manuals.
+        # The function setup_manuals will setup most files --minimum-perc=75
+        # for most files.
+        tx pull -a -f
 
-# Pull upstream translations of all downloaded files but do not
-# download new files.
-# Use lower percentage here to update the existing files.
-tx pull -f --minimum-perc=50
+        # Pull upstream translations of all downloaded files but do not
+        # download new files.
+        # Use lower percentage here to update the existing files.
+        tx pull -f --minimum-perc=50
+        ;;
+    Zanata)
+        pull_from_zanata_manuals "$PROJECT"
+        ;;
+esac
 
 # Compress downloaded po files
 # Only touch glossary in openstack-manuals but not in any other