Prepare new translation functions for python projects

python-neutronclient and python-novaclient repos should work with
the new scripts
(though the current locale files still exist in the repo).
The current script will be replaced after the migration completes.

This script supports $modulename/locale/$modulename.pot as POT filename.

Part of infra-spec Improve Translation Setup

Story: 2000452
Change-Id: I67bfab533a8fd48f038c10e82509a2e626205a33

jenkins/scripts/common_translation_update.sh

@@ -252,6 +252,25 @@ function extract_messages {
     python setup.py $QUIET extract_messages --keyword "_C:1c,2 _P:1,2"
 }
 
+# TODO(amotoki): Finally this should replace current extract_messages.
+function extract_messages_new {
+    local modulename=$1
+
+    # NOTE(amotoki): POT file == $modulename/locale/$modulename.pot
+    local POT=${modulename}/locale/${modulename}.pot
+
+    # In case this is an initial run, the locale directory might not
+    # exist, so create it since extract_messages will fail if it does
+    # not exist. So, create it if needed.
+    mkdir -p ${modulename}/locale
+
+    # Update the .pot files
+    # The "_C" and "_P" prefix are for more-gettext-support blueprint,
+    # "_C" for message with context, "_P" for plural form message.
+    python setup.py $QUIET extract_messages --keyword "_C:1c,2 _P:1,2" \
+        --output-file ${POT}
+}
+
 # Run extract_messages for log messages.
 # Needs variables setup via setup_loglevel_vars.
 function extract_messages_log {
@@ -275,6 +294,29 @@ function extract_messages_log {
     done
 }
 
+# TODO(amotoki): Finally this should replace current extract_messages_log.
+function extract_messages_log_new {
+    local modulename=$1
+    local POT
+    local trans
+
+    # Update the .pot files
+    for level in $LEVELS ; do
+        # NOTE(amotoki): POT file == $modulename/locale/$modulename.pot
+        POT=${modulename}/locale/${modulename}-log-${level}.pot
+        python setup.py $QUIET extract_messages --no-default-keywords \
+            --keyword ${LKEYWORD[$level]} \
+            --output-file ${POT}
+        # We don't need to add or send around empty source files.
+        trans=$(msgfmt --statistics -o /dev/null ${POT} 2>&1)
+        if [ "$trans" = "0 translated messages." ] ; then
+            rm $POT
+            # Remove file from git if it's under version control.
+            git rm --ignore-unmatch $POT
+        fi
+    done
+}
+
 # TODO(amotoki): After we use $modulename/locale/$modulename.pot
 # for normal python projects, this function will be used both
 # by python and django projects. It should be renamed to setup_project.
@@ -423,9 +465,9 @@ function check_po_file {
 # Remove obsolete files. We might have added them in the past but
 # would not add them today, so let's eventually remove them.
 function cleanup_po_files {
-    local project=$1
+    local modulename=$1
 
-    for i in $(find $project/locale -name *.po) ; do
+    for i in $(find $modulename/locale -name *.po) ; do
         check_po_file "$i"
         if [ $RATIO -lt 20 ]; then
             git rm -f $i

jenkins/scripts/propose_translation_update.sh

@@ -100,6 +100,36 @@ function propose_python {
     git add $PROJECT/locale/*
 }
 
+# TODO(amotoki): Finally this should replace current propose_python.
+function propose_python_new {
+    local project=$1
+    local modulename=$2
+
+    # Pull updated translations from Zanata
+    pull_from_zanata "$project"
+
+    # Extract all messages from project, including log messages.
+    extract_messages_new "$modulename"
+    extract_messages_log_new "$modulename"
+
+    # Now add all changed files to git.
+    # Note we add them here to not have to differentiate in the functions
+    # between new files and files already under git control.
+    git add $modulename/locale/*
+
+    # Remove obsolete files.
+    cleanup_po_files "$modulename"
+
+    # Compress downloaded po files, this needs to be done after
+    # cleanup_po_files since that function needs to have information the
+    # number of untranslated strings.
+    compress_po_files "$modulename"
+
+    # Some files were changed, add changed files again to git, so that we
+    # can run git diff properly.
+    git add $modulename/locale/*
+}
+
 function propose_horizon {
 
     # Pull updated translations from Zanata.
@@ -180,14 +210,21 @@ case "$PROJECT" in
         propose_horizon
         ;;
     # Test of translation setup improvement
-    murano-dashboard|magnum-ui)
-        # TODO(amotoki): Honor module name in propose_*
-        # MODULENAME=$(get_modulename $PROJECT python)
-        # if [ -n "$MODULENAME" ]; then
-        #     setup_project "$PROJECT" "$ZANATA_VERSION"
-        #     setup_loglevel_vars
-        #     propose_python
-        # fi
+    murano-dashboard|magnum-ui|python-neutronclient|python-novaclient)
+        # ---- Python projects ----
+        # NOTE: At now POT file == $modulename/locale/$modulename.pot
+        #       so this script works.
+        # TODO(amotoki):
+        # * Move POT/PO file to $modulename/locale/$modulename.pot
+        # * Update setup.cfg (babel related)
+        # * Rename Zanata resource
+        MODULENAME=$(get_modulename $PROJECT python)
+        if [ -n "$MODULENAME" ]; then
+            setup_django "$PROJECT" "$MODULENAME" "$ZANATA_VERSION"
+            propose_python_new "$PROJECT" "$MODULENAME"
+        fi
+
+        # ---- Django projects ----
         MODULENAME=$(get_modulename $PROJECT django)
         if [ -n "$MODULENAME" ]; then
             setup_django "$PROJECT" "$MODULENAME" "$ZANATA_VERSION"

jenkins/scripts/upstream_translation_update.sh

@@ -55,15 +55,23 @@ case "$PROJECT" in
         ./run_tests.sh --makemessages -V
         ;;
     # Test of translation setup improvement
-    murano-dashboard|magnum-ui)
-        # TODO(amotoki): Honor module name in extract_*
-        # MODULENAME=$(get_modulename $PROJECT python)
-        # if [ -n "$MODULENAME" ]; then
-        #     setup_project "$PROJECT" "$ZANATA_VERSION"
-        #     setup_loglevel_vars
-        #     extract_messages
-        #     extract_messages_log "$PROJECT"
-        # fi
+    murano-dashboard|magnum-ui|python-neutronclient|python-novaclient)
+        # ---- Python projects ----
+        # NOTE: At now POT file == $modulename/locale/$modulename.pot
+        #       so this script works.
+        # TODO(amotoki):
+        # * Move POT/PO file to $modulename/locale/$modulename.pot
+        # * Update setup.cfg (babel related)
+        # * Rename Zanata resource
+        MODULENAME=$(get_modulename $PROJECT python)
+        if [ -n "$MODULENAME" ]; then
+            setup_django "$PROJECT" "$MODULENAME" "$ZANATA_VERSION"
+            setup_loglevel_vars
+            extract_messages_new "$MODULENAME"
+            extract_messages_log_new "$MODULENAME"
+        fi
+
+        # ---- Django projects ----
         MODULENAME=$(get_modulename $PROJECT django)
         if [ -n "$MODULENAME" ]; then
             setup_django "$PROJECT" "$MODULENAME" "$ZANATA_VERSION"