project-config/jenkins/scripts/common_translation_update.sh

#!/bin/bash -xe
# Common code used by propose_translation_update.sh and
# upstream_translation_update.sh

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# Initial transifex setup
function setup_translation {
    # Track in HAS_CONFIG whether we run "tx init" since calling it
    # will add the file .tx/config - and "tx set" might update it. If
    # "tx set" updates .tx/config, we need to handle the file if it
    # existed before.
    HAS_CONFIG=1

    # Initialize the transifex client, if there's no .tx directory
    if [ ! -d .tx ] ; then
        tx init --host=https://www.transifex.com
        HAS_CONFIG=0
    fi
}

# Setup a project for transifex
function setup_project {
    local project=$1

    # Transifex project name does not include "."
    tx_project=${project/\./}
    tx set --auto-local -r ${tx_project}.${tx_project}-translations \
        "${project}/locale/<lang>/LC_MESSAGES/${project}.po" \
        --source-lang en \
        --source-file ${project}/locale/${project}.pot -t PO \
        --execute
}

# Setup project horizon for transifex
function setup_horizon {
    local project=horizon

    # Horizon JavaScript Translations
    tx set --auto-local -r ${project}.${project}-js-translations \
        "${project}/locale/<lang>/LC_MESSAGES/djangojs.po" \
        --source-lang en \
        --source-file ${project}/locale/en/LC_MESSAGES/djangojs.po \
        -t PO --execute

    # Horizon Translations
    tx set --auto-local -r ${project}.${project}-translations \
        "${project}/locale/<lang>/LC_MESSAGES/django.po" \
        --source-lang en \
        --source-file ${project}/locale/en/LC_MESSAGES/django.po \
        -t PO --execute

    # OpenStack Dashboard Translations
    tx set --auto-local -r ${project}.openstack-dashboard-translations \
        "openstack_dashboard/locale/<lang>/LC_MESSAGES/django.po" \
        --source-lang en \
        --source-file openstack_dashboard/locale/en/LC_MESSAGES/django.po \
        -t PO --execute
}

# Set global variable DocFolder for manuals projects
function init_manuals {
    project=$1

    DocFolder="doc"
    if [ $project = "api-site" -o $project = "security-doc" ] ; then
        DocFolder="./"
    fi
}

# Setup project manuals projects (api-site, openstack-manuals,
# operations-guide) for transifex
function setup_manuals {
    local project=$1
    # Generate pot one by one
    for FILE in ${DocFolder}/*; do
        # Skip non-directories
        if [ ! -d $FILE ]; then
            continue
        fi
        DOCNAME=${FILE#${DocFolder}/}
        # Ignore directories that will not get translated
        if [[ "$DOCNAME" =~ ^(www|tools|generated|publish-docs)$ ]]; then
            continue
        fi
        # Ignore directories starting with playground
        if [[ "$DOCNAME" =~ "^playground" ]]; then
            continue
        fi
        # Skip glossary in all repos besides openstack-manuals.
        if [ "$project" != "openstack-manuals" -a "$DOCNAME" == "glossary" ]; then
            continue
        fi
        # Minimum amount of translation done, 75 % by default.
        PERC=75
        if [ "$project" == "openstack-manuals" ]; then
            # The common and glossary directories are used by the
            # other guides, let's be more liberal here since teams
            # might only translate the files used by a single
            # guide. We use 8 % since that downloads the currently
            # translated files.
            if [ "$DOCNAME" == "common" -o "$DOCNAME" == "glossary" ]; then
                PERC=8
            fi
        fi
        # Update the .pot file
        ./tools/generatepot ${DOCNAME}
        if [ -f ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot ]; then
            # Add all changed files to git
            git add ${DocFolder}/${DOCNAME}/locale/*
            # Set auto-local
            tx set --auto-local -r openstack-manuals-i18n.${DOCNAME} \
                "${DocFolder}/${DOCNAME}/locale/<lang>.po" --source-lang en \
                --source-file ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot \
                --minimum-perc=$PERC \
                -t PO --execute
        fi
    done

}

# Setup git so that git review works
function setup_git {
    git config user.name "OpenStack Proposal Bot"
    git config user.email "openstack-infra@lists.openstack.org"
    git config gitreview.username "proposal-bot"
}

# Setup project so that git review works, sets global variable
# COMMIT_MSG.
function setup_review {
    FULL_PROJECT=$(grep project .gitreview  | cut -f2 -d= |sed -e 's/\.git$//')
    set +e
    read -d '' COMMIT_MSG <<EOF
Imported Translations from Transifex

For more information about this automatic import see:
https://wiki.openstack.org/wiki/Translations/Infrastructure
EOF
    set -e
    git review -s

    # See if there is an open change in the transifex/translations
    # topic. If so, get the change id for the existing change for use
    # in the commit msg.
    change_info=`ssh -p 29418 proposal-bot@review.openstack.org gerrit query --current-patch-set status:open project:$FULL_PROJECT topic:transifex/translations owner:proposal-bot`
    previous=`echo "$change_info" | grep "^  number:" | awk '{print $2}'`
    if [ "x${previous}" != "x" ] ; then
        change_id=`echo "$change_info" | grep "^change" | awk '{print $2}'`
        # Read returns a non zero value when it reaches EOF. Because we use a
        # heredoc here it will always reach EOF and return a nonzero value.
        # Disable -e temporarily to get around the read.
        set +e
        read -d '' COMMIT_MSG <<EOF
Imported Translations from Transifex

For more information about this automatic import see:
https://wiki.openstack.org/wiki/Translations/Infrastructure

Change-Id: $change_id
EOF
        set -e
    fi
    # If the open change an already approved, let's not queue a new
    # patch but let's merge the other patch first.
    # This solves the problem that when the gate pipeline backup
    # reaches roughly a day, no matter how quickly you approve the new
    # update it will always get sniped out of the gate by another.
    # It also helps, when you approve close to the time this job is
    # run.
    if [ "x${previous}" != "x" ] ; then
        # Use the JSON format since it is very compact and easy to grep
        change_info=`ssh -p 29418 proposal-bot@review.openstack.org gerrit query --current-patch-set --format=JSON $change_id`
        # Check for:
        # 1) Workflow approval (+1)
        # 2) no -1/-2 by Jenkins
        # 3) no -2 by reviewers
        # 4) no Workflow -1 (WIP)
        #
        if  `echo $change_info|grep -q '{"type":"Workflow","description":"Workflow","value":"1"'` \
            && ! `echo $change_info|grep -q '{"type":"Verified","description":"Verified","value":"-[12]","grantedOn":[0-9]*,"by":{"name":"Jenkins","username":"jenkins"}}'`  \
            && ! `echo $change_info|grep -q '{"type":"Code-Review","description":"Code-Review","value":"-2"'` \
            && ! `echo $change_info|grep -q '{"type":"Workflow","description":"Workflow","value":"-1"'`  ; then
            echo "Job already approved, exiting"
            exit 0
        fi
    fi
}

# Propose patch using COMMIT_MSG
function send_patch {

    # Revert any changes done to .tx/config
    if [ $HAS_CONFIG -eq 1 ]; then
        git reset -q .tx/config
        git checkout -- .tx/config
    fi

    # Don't send a review if nothing has changed.
    if [ `git diff --cached |wc -l` -gt 0 ]; then
        # Commit and review
        git commit -F- <<EOF
$COMMIT_MSG
EOF
        git review -t transifex/translations

    fi
}

# Setup global variables LEVELS and LKEYWORDS
function setup_loglevel_vars {
    # Strings for various log levels
    LEVELS="info warning error critical"
    # Keywords for each log level:
    declare -g -A LKEYWORD
    LKEYWORD['info']='_LI'
    LKEYWORD['warning']='_LW'
    LKEYWORD['error']='_LE'
    LKEYWORD['critical']='_LC'
}

# Setup transifex configuration for log level message translation.
# Needs variables setup via setup_loglevel_vars.
function setup_loglevel_project {
    project=$1

    # Transifex project name does not include "."
    tx_project=${project/\./}

    for level in $LEVELS ; do
        # Bootstrapping: Create file if it does not exist yet,
        # otherwise "tx set" will fail.
        if [ ! -e  ${project}/locale/${project}-log-${level}.pot ]; then
            touch ${project}/locale/${project}-log-${level}.pot
        fi
        tx set --auto-local -r ${tx_project}.${tx_project}-log-${level}-translations \
            "${project}/locale/<lang>/LC_MESSAGES/${project}-log-${level}.po" \
            --source-lang en \
            --source-file ${project}/locale/${project}-log-${level}.pot -t PO \
            --execute
    done
}

# Run extract_messages for user visible messages and log messages.
# Needs variables setup via setup_loglevel_vars.
function extract_messages_log {
    project=$1

    # Update the .pot files
    python setup.py extract_messages
    for level in $LEVELS ; do
        python setup.py extract_messages --no-default-keywords \
            --keyword ${LKEYWORD[$level]} \
            --output-file ${project}/locale/${project}-log-${level}.pot
    done
}

# Setup project django_openstack_auth for transifex
function setup_django_openstack_auth {
    tx set --auto-local -r horizon.djangopo \
        "openstack_auth/locale/<lang>/LC_MESSAGES/django.po" \
        --source-lang en \
        --source-file openstack_auth/locale/openstack_auth.pot -t PO \
        --execute
}

# Filter out files that we do not want to commit
function filter_commits {
    # Don't add new empty files.
    for f in `git diff --cached --name-only --diff-filter=A`; do
        # Files should have at least one non-empty msgid string.
        if ! grep -q 'msgid "[^"]' "$f" ; then
            git reset -q "$f"
            rm "$f"
        fi
    done

    # Don't send files where the only things which have changed are
    # the creation date, the version number, the revision date,
    # comment lines, or diff file information.
    # Also, don't send files if only .pot files would be changed.
    PO_CHANGE=0
    for f in `git diff --cached --name-only`; do
        # It's ok if the grep fails
        set +e
        changed=$(git diff --cached "$f" \
            | egrep -v "(POT-Creation-Date|Project-Id-Version|PO-Revision-Date)" \
            | egrep -c "^([-+][^-+#])")
        set -e
        if [ $changed -eq 0 ]; then
            git reset -q "$f"
            git checkout -- "$f"
        # Check for all files endig with ".po"
        elif [[ $f =~ .po$ ]] ; then
            PO_CHANGE=1
        fi
    done
    # If no po file was changed, only pot source files were changed
    # and those changes can be ignored as they give no benefit on
    # their own.
    if [ $PO_CHANGE -eq 0 ] ; then
        for f in `git diff --cached --name-only` ; do
            git reset -q "$f"
            git checkout -- "$f"
        done
    fi
}

# Remove obsolete files. We might have added them in the past but
# would not add them today, so let's eventually remove them.
function cleanup_po_files {
    local project=$1

    for i in `find $project/locale -name *.po `; do
        # Output goes to stderr, so redirect to stdout to catch it.
        trans=`msgfmt --statistics -o /dev/null $i 2>&1`
        check="^0 translated messages"
        if [[ $trans =~ $check ]] ; then
            # Nothing is translated, remove the file.
            git rm -f $i
        else
            if [[ $trans =~ " translated message" ]] ; then
                trans_no=`echo $trans|sed -e 's/ translated message.*$//'`
            else
                trans_no=0
            fi
            if [[ $trans =~ " untranslated message" ]] ; then
                untrans_no=`echo $trans|sed -e 's/^.* \([0-9]*\) untranslated message.*/\1/'`
            else
                untrans_no=0
            fi
            let total=$trans_no+$untrans_no
            let ratio=100*$trans_no/$total
            # Since we only download files that are at least
            # translated to 75 per cent, let's delete those that have
            # signficantly less translations.
            # For now we delete files that suddenly are less than 20
            # per cent translated.
            if [[ "$ratio" -lt "20" ]] ; then
                git rm -f $i
            fi
        fi
    done
}