Merge "Create migration tools for setting up the environment"
This commit is contained in:
Zuul
105
tools/migration/README.rst
Normal file
105
tools/migration/README.rst
Normal file
@@ -0,0 +1,105 @@
|
||||
=======================================
|
||||
Migration Tools from Zanata to Weblate
|
||||
=======================================
|
||||
|
||||
This folder provides tools to migrate translation projects from Zanata
|
||||
to Weblate.
|
||||
|
||||
.. note:
|
||||
|
||||
The tool currently works with horizon and plugin projects.
|
||||
|
||||
Background
|
||||
----------
|
||||
|
||||
OpenStack I18n team has been using
|
||||
`Zanata <https://github.com/zanata/zanata-platform>`__
|
||||
as translation platform, which development and release is discountinued
|
||||
from August 2018.
|
||||
To ensure continued translation management and improve OpenStack
|
||||
internationalization workflow, OpenStack I18n SIG is migrating
|
||||
Zanata projects to Weblate.
|
||||
|
||||
Objectives
|
||||
----------
|
||||
|
||||
* Preserve existing translation structure and format as much as possible
|
||||
|
||||
Migration Workflow
|
||||
------------------
|
||||
|
||||
1. Set up the workspace.
|
||||
2. Generate POT files from cloned project repositories.
|
||||
3. Create ``zanata.xml`` and export translations (PO files) from Zanata.
|
||||
4. Create a Weblate project, category, and component.
|
||||
5. Create translations and upload a translation file for each locale.
|
||||
|
||||
How to use
|
||||
----------
|
||||
|
||||
1. Clone the repository
|
||||
2. Run the migration script:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
./migration_resources.sh <project_name> <version> <workspace_name>
|
||||
|
||||
Arguments:
|
||||
|
||||
* project_name: The name of the OpenStack project to migrate.
|
||||
* version: The version of the project. The default is "master".
|
||||
e.g. stable-2025.1
|
||||
* workspace_name: The folder name for the migration workspace.
|
||||
It will be installed in the home directory. The default is "workspace".
|
||||
|
||||
Folder and File Structure
|
||||
--------------
|
||||
.. code-block:: text
|
||||
|
||||
├── setup_env/
|
||||
└── migration_resources.sh
|
||||
|
||||
* setup_env/: Scripts to create a virtual environment and install dependencies.
|
||||
These scripts also create a workspace folder for migration tasks.
|
||||
* migration_resources.sh: Main script to migrate translation resources.
|
||||
They include various modules to perform actual tasks for resource
|
||||
migration step by step.
|
||||
|
||||
.. note::
|
||||
|
||||
The current tool mainly focuses on translation resource migration,
|
||||
and actual resource migration process currently requires careful
|
||||
supervision to make sure that each migration step is performed
|
||||
well or not.
|
||||
|
||||
Workspace Structure
|
||||
-------------------
|
||||
|
||||
By default, the migration workspace is installed to your home directory
|
||||
with the following folder structure:
|
||||
|
||||
.. note::
|
||||
|
||||
You can change the base directory by manually setting WORK_DIR
|
||||
on setup_env/setup.sh.
|
||||
|
||||
* .venv/: Python virtual environment containing migration dependencies
|
||||
* projects/: Migration workspace
|
||||
|
||||
* <project_name>/: Project-specific workspace
|
||||
|
||||
* <cloned_project_name>/: Cloned project repository
|
||||
* pot/: POT files for each component
|
||||
* translations/: Exported translations from Zanata
|
||||
|
||||
Directory Layout::
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
<workspace_name>/
|
||||
├── .venv/
|
||||
└── projects/
|
||||
└── <project_name>/
|
||||
├── <cloned_project_name>/
|
||||
├── pot/
|
||||
└── translations/
|
||||
50
tools/migration/migration_resources.sh
Executable file
50
tools/migration/migration_resources.sh
Executable file
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
# Migration script to create Weblate components and translations
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
PROJECT=$1
|
||||
BRANCH_NAME=${2:-"master"}
|
||||
WORKSPACE_NAME=${3:-"workspace"}
|
||||
|
||||
# Replace /'s in branch names with -'s because Zanata doesn't
|
||||
# allow /'s in version names.
|
||||
ZANATA_VERSION=${BRANCH_NAME//\//-}
|
||||
|
||||
# List the components to be handled
|
||||
COMPONENTS=()
|
||||
|
||||
SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
source $SCRIPTSDIR/setup_env/setup.sh
|
||||
|
||||
# We need a UTF-8 locale, set it properly in case it's not set.
|
||||
export LANG=en_US.UTF-8
|
||||
|
||||
# You should set WEBLATE_URL and WEBLATE_TOKEN
|
||||
# system environment variables.
|
||||
echo "[INFO] Check variables"
|
||||
if [ -z "$WEBLATE_URL" ] || [ "$WEBLATE_URL" == "<weblate_url>" ]; then
|
||||
echo "[ERROR] WEBLATE_URL is not set"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$WEBLATE_TOKEN" ] || [ "$WEBLATE_TOKEN" == "<weblate_token>" ]; then
|
||||
echo "[ERROR] WEBLATE_TOKEN is not set"
|
||||
exit 1
|
||||
fi
|
||||
echo "[INFO] WEBLATE_URL and WEBLATE_TOKEN are set"
|
||||
|
||||
echo "[INFO] Setup environment and prepare workspace"
|
||||
if ! setup_env_and_prepare_workspace "$PROJECT"; then
|
||||
echo "[ERROR] Failed to setup environment and prepare workspace"
|
||||
exit 1
|
||||
fi
|
||||
5
tools/migration/setup_env/bindep.txt
Normal file
5
tools/migration/setup_env/bindep.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
# This is a cross-platform list tracking distribution packages needed by tests;
|
||||
# see https://docs.openstack.org/infra/bindep/ for additional information.
|
||||
|
||||
gettext
|
||||
jq
|
||||
3
tools/migration/setup_env/requirements.txt
Normal file
3
tools/migration/setup_env/requirements.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
bindep
|
||||
polib==1.2.0
|
||||
lxml==5.3.0
|
||||
169
tools/migration/setup_env/setup.sh
Normal file
169
tools/migration/setup_env/setup.sh
Normal file
@@ -0,0 +1,169 @@
|
||||
#!/bin/bash
|
||||
# Set venv and folder structure for migration
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
WORK_DIR="$HOME/$WORKSPACE_NAME"
|
||||
|
||||
function create_python_venv() {
|
||||
if ! command -v python3 &> /dev/null; then
|
||||
echo "[ERROR] Python 3 is not installed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# create venv
|
||||
if [ ! -d "$WORK_DIR/.venv" ]; then
|
||||
python3 -m venv "$WORK_DIR/.venv" >/dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "[ERROR] Failed to create virtual environment"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[INFO] Virtual environment created"
|
||||
return 0
|
||||
}
|
||||
|
||||
function install_dependencies() {
|
||||
source "$WORK_DIR/.venv/bin/activate"
|
||||
|
||||
cd "$SCRIPTSDIR/setup_env"
|
||||
# Install python dependencies
|
||||
pip3 install -r requirements.txt
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "[ERROR] Failed to install requirements.txt in venv."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Install dependencies with bindep
|
||||
bindep_packages=$("$WORK_DIR/.venv/bin/bindep" -b -f bindep.txt 2>/dev/null)
|
||||
if [ -n "$bindep_packages" ]; then
|
||||
echo "[INFO] Installing system dependencies with bindep: $bindep_packages"
|
||||
sudo apt install -y $bindep_packages
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "[ERROR] Failed to install system dependencies with bindep"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function check_zanata_cli() {
|
||||
# Check zanata-cli is installed
|
||||
if ! command -v zanata-cli &> /dev/null; then
|
||||
echo "[ERROR] zanata-cli is not installed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check zanata.ini file exists
|
||||
if [ ! -f "$HOME/.config/zanata.ini" ]; then
|
||||
echo "[ERROR] zanata.ini is not found"
|
||||
return 1
|
||||
fi
|
||||
echo "[INFO] zanata-cli is installed and zanata.ini file exists"
|
||||
return 0
|
||||
}
|
||||
|
||||
function setup_env() {
|
||||
if ! prepare_workspace; then
|
||||
echo "[ERROR] Failed to prepare workspace."
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! create_python_venv; then
|
||||
echo "[ERROR] Failed to create python venv"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! install_dependencies; then
|
||||
echo "[ERROR] Failed to install dependencies."
|
||||
echo "[ERROR] Please check bindep.txt and requirements.txt"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! check_zanata_cli; then
|
||||
echo "[ERROR] Failed to check zanata-cli."
|
||||
echo "[ERROR] Please check if zanata-cli is installed and zanata.ini file exists."
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "[INFO] Environment setup successfully"
|
||||
return 0
|
||||
}
|
||||
|
||||
function prepare_workspace() {
|
||||
# Create workspace directory
|
||||
if [ ! -d "$WORK_DIR" ]; then
|
||||
if ! mkdir -p "$WORK_DIR"; then
|
||||
echo "[ERROR] Failed to create $WORKSPACE_NAME directory"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create projects directory
|
||||
if [ ! -d "$WORK_DIR/projects" ]; then
|
||||
if ! mkdir -p "$WORK_DIR/projects"; then
|
||||
echo "[ERROR] Failed to create projects directory"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[INFO] Workspace directory created successfully"
|
||||
return 0
|
||||
|
||||
}
|
||||
|
||||
function prepare_project_workspace() {
|
||||
local project=$1
|
||||
|
||||
# Create project directory
|
||||
if [ ! -d "$WORK_DIR/projects/$project" ]; then
|
||||
mkdir -p $WORK_DIR/projects/$project
|
||||
fi
|
||||
|
||||
# Create pot directory
|
||||
if [ ! -d "$WORK_DIR/projects/$project/pot" ]; then
|
||||
mkdir -p $WORK_DIR/projects/$project/pot
|
||||
echo "[INFO] Pot directory created successfully"
|
||||
fi
|
||||
|
||||
# Create translations directory
|
||||
if [ ! -d "$WORK_DIR/projects/$project/translations" ]; then
|
||||
mkdir -p $WORK_DIR/projects/$project/translations
|
||||
echo "[INFO] Translations directory created successfully"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function setup_env_and_prepare_workspace() {
|
||||
local project=$1
|
||||
|
||||
# Create a virtual environment and install system dependencies
|
||||
echo "[INFO] Setup the virtual environment and install system dependencies"
|
||||
if ! setup_env; then
|
||||
echo "[ERROR] Failed to setup environment"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Prepare workspace folders for migration tasks
|
||||
echo "[INFO] Prepare workspace folders"
|
||||
if ! prepare_workspace; then
|
||||
echo "[ERROR] Failed to prepare workspace"
|
||||
return 1
|
||||
fi
|
||||
|
||||
prepare_project_workspace "$project"
|
||||
return 0
|
||||
}
|
||||
9
tools/migration/tox.ini
Normal file
9
tools/migration/tox.ini
Normal file
@@ -0,0 +1,9 @@
|
||||
[tox]
|
||||
minversion = 3.1.0
|
||||
envlist= py{39,310,311,312,313}
|
||||
skipdist = True
|
||||
|
||||
[testenv]
|
||||
deps = -r {toxinidir}/setup_env/requirements.txt
|
||||
commands =
|
||||
pip check
|
||||
Reference in New Issue
Block a user