Add new django extraction

- Create a new extract_django function that can be used
  for all django repositories.
- Create a new setup_django function to setup translation server
  for django repositories.
- Create a new propose_django function to propose translations.
- Extra a module name from setup.cfg.

Use these functions for murano-dashboard.

Co-Authored-By: Akihiro Motoki <amotoki@gmail.com>
Change-Id: I75c61ca709e05740c8932218b132752828b90b91
Story: 2000452

jenkins/scripts/common_translation_update.sh

@@ -22,6 +22,15 @@ TOPIC=zanata/translations
 # Used for setup.py babel commands
 QUIET="--quiet"
 
+# Get a module name of a project
+function get_modulename {
+    local project=$1
+    local target=$2
+
+    /usr/local/jenkins/slave_scripts/get-modulename.py \
+        -p $project -t $target
+}
+
 # Setup a project for Zanata
 function setup_project {
     local project=$1
@@ -266,6 +275,59 @@ function extract_messages_log {
     done
 }
 
+# TODO(amotoki): After we use $modulename/locale/$modulename.pot
+# for normal python projects, this function will be used both
+# by python and django projects. It should be renamed to setup_project.
+# Setup django project for Zanata
+function setup_django {
+    local project=$1
+    local modulename=$2
+    local version=${3:-master}
+
+    /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
+        -p $project -v $version --srcdir $modulename/locale \
+        --txdir $modulename/locale -r '**/*.pot' \
+        '{locale_with_underscore}/LC_MESSAGES/{filename}.po' -f zanata.xml
+}
+
+# Extract messages for a django project, we need to update django.pot
+# and djangojs.pot.
+function extract_messages_django {
+    local modulename=$1
+
+    # We need to install horizon
+    VENV=$(mktemp -d)
+    trap "rm -rf $VENV" EXIT
+    virtualenv $VENV
+
+    # TODO(jaegerandi): Switch to zuul-cloner once it's safe to use
+    # zuul-cloner in post jobs and we have a persistent cache on
+    # proposal node.
+    root=$(mktemp -d)
+    trap "rm -rf $VENV $root" EXIT
+
+    git clone --depth=1 git://git.openstack.org/openstack/horizon.git $root/horizon
+    (cd ${root}/horizon && $VENV/bin/pip install .)
+
+    # Horizon has these as dependencies but let's be sure.
+    # TODO(amotoki): Pull required versions from g-r.
+    $VENV/bin/pip install Babel django-babel
+    KEYWORDS="-k gettext_noop -k gettext_lazy -k ngettext_lazy:1,2"
+    KEYWORDS+=" -k ugettext_noop -k ugettext_lazy -k ungettext_lazy:1,2"
+    KEYWORDS+=" -k npgettext:1c,2,3 -k pgettext_lazy:1c,2 -k npgettext_lazy:1c,2,3"
+
+    for DOMAIN in djangojs django ; do
+        if [ -f babel-${DOMAIN}.cfg ]; then
+            mkdir -p ${modulename}/locale
+            touch ${modulename}/locale/${DOMAIN}.pot
+            $VENV/bin/pybabel extract -F babel-${DOMAIN}.cfg \
+                -o ${modulename}/locale/${DOMAIN}.pot $KEYWORDS ${modulename}
+        fi
+    done
+    rm -rf $VENV $root
+    trap "" EXIT
+}
+
 # Setup project django_openstack_auth for Zanata
 function setup_django_openstack_auth {
     local project=django_openstack_auth

jenkins/scripts/get-modulename.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import ConfigParser as configparser
+import os
+
+
+DJANGO_PROJECT_SUFFIXES = ('-dashboard',
+                           '-horizon',
+                           '-ui',
+                           'django_openstack_auth',
+)
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description='Find module names in a repository.')
+    parser.add_argument('-p', '--project', required=True)
+    parser.add_argument('-t', '--target',
+                        choices=['python', 'django'],
+                        default='python',
+                        help='Type of modules to search (default: python).')
+    parser.add_argument('-f', '--file', default='setup.cfg',
+                        help='Path of setup.cfg file.')
+    return parser.parse_args()
+
+
+def split_multiline(value):
+    value = [element for element in
+             (line.strip() for line in value.split('\n'))
+             if element]
+    return value
+
+
+def read_config(path):
+    parser = configparser.SafeConfigParser()
+    parser.read(path)
+
+    config = {}
+    for section in parser.sections():
+        config[section] = dict(parser.items(section))
+    return config
+
+
+def get_option(config, section, option, multiline=False):
+    if section not in config:
+        return
+    value = config[section].get(option)
+    if multiline:
+        value = split_multiline(value)
+    return value
+
+
+def get_translate_options(config, target):
+    translate_options = {}
+    for key, value in config['openstack_translations'].items():
+        values = split_multiline(value)
+        if values:
+            translate_options[key] = values
+    return translate_options.get('%s_modules' % target, [])
+
+
+def get_valid_modules(config, project, target):
+    is_django = any(project.endswith(suffix)
+                    for suffix in DJANGO_PROJECT_SUFFIXES)
+    if is_django != (target == 'django'):
+        return []
+
+    return get_option(config, 'files', 'packages',
+                      multiline=True)
+
+
+def main():
+    args = get_args()
+    config = read_config(args.file)
+
+    if 'openstack_translations' in config:
+        translate_options = get_translate_options(config, args.target)
+        return translate_options
+
+    modules = get_valid_modules(config, args.project, args.target)
+
+    # print(' '.join(modules))
+    # A shortest module name is selected now.
+    if modules:
+        print(sorted(modules, key=len)[0])
+
+if __name__ == '__main__':
+    main()

jenkins/scripts/propose_translation_update.sh

@@ -148,6 +148,23 @@ function propose_magnum_ui {
     git add magnum_ui/locale/*
 }
 
+# This function can be used for all django projects
+function propose_django {
+    local project=$1
+    local modulename=$2
+    # Pull updated translations from Zanata.
+    pull_from_zanata "$project"
+
+    # Update the .pot file
+    extract_messages_django "$modulename"
+
+    # Compress downloaded po files
+    compress_po_files "$modulename"
+
+    # Add all changed files to git
+    git add $modulename/locale/*
+}
+
 # Setup git repository for git review.
 setup_git
 
@@ -176,6 +193,21 @@ case "$PROJECT" in
         setup_magnum_ui "$ZANATA_VERSION"
         propose_magnum_ui
         ;;
+    # Test of translation setup improvement
+    murano-dashboard)
+        # TODO(amotoki): Honor module name in propose_*
+        # MODULENAME=$(get_modulename $PROJECT python)
+        # if [ -n "$MODULENAME" ]; then
+        #     setup_project "$PROJECT" "$ZANATA_VERSION"
+        #     setup_loglevel_vars
+        #     propose_python
+        # fi
+        MODULENAME=$(get_modulename $PROJECT django)
+        if [ -n "$MODULENAME" ]; then
+            setup_django "$PROJECT" "$MODULENAME" "$ZANATA_VERSION"
+            propose_django "$PROJECT" "$MODULENAME"
+        fi
+        ;;
     *)
         # Project specific setup.
         setup_project "$PROJECT" "$ZANATA_VERSION"

jenkins/scripts/upstream_translation_update.sh

@@ -52,6 +52,22 @@ case "$PROJECT" in
         setup_magnum_ui "$ZANATA_VERSION"
         ./run_tests.sh --makemessages -V
         ;;
+    # Test of translation setup improvement
+    murano-dashboard)
+        # TODO(amotoki): Honor module name in extract_*
+        # MODULENAME=$(get_modulename $PROJECT python)
+        # if [ -n "$MODULENAME" ]; then
+        #     setup_project "$PROJECT" "$ZANATA_VERSION"
+        #     setup_loglevel_vars
+        #     extract_messages
+        #     extract_messages_log "$PROJECT"
+        # fi
+        MODULENAME=$(get_modulename $PROJECT django)
+        if [ -n "$MODULENAME" ]; then
+            setup_django "$PROJECT" "$MODULENAME" "$ZANATA_VERSION"
+            extract_messages_django "$MODULENAME"
+        fi
+        ;;
     *)
         setup_project "$PROJECT" "$ZANATA_VERSION"
         setup_loglevel_vars