diff --git a/tools/migration/README.rst b/tools/migration/README.rst new file mode 100644 index 0000000..66078fe --- /dev/null +++ b/tools/migration/README.rst @@ -0,0 +1,105 @@ +======================================= +Migration Tools from Zanata to Weblate +======================================= + +This folder provides tools to migrate translation projects from Zanata +to Weblate. + +.. note: + + The tool currently works with horizon and plugin projects. + +Background +---------- + +OpenStack I18n team has been using +`Zanata `__ +as translation platform, which development and release is discountinued +from August 2018. +To ensure continued translation management and improve OpenStack +internationalization workflow, OpenStack I18n SIG is migrating +Zanata projects to Weblate. + +Objectives +---------- + +* Preserve existing translation structure and format as much as possible + +Migration Workflow +------------------ + +1. Set up the workspace. +2. Generate POT files from cloned project repositories. +3. Create ``zanata.xml`` and export translations (PO files) from Zanata. +4. Create a Weblate project, category, and component. +5. Create translations and upload a translation file for each locale. + +How to use +---------- + +1. Clone the repository +2. Run the migration script: + +.. code-block:: bash + + ./migration_resources.sh + +Arguments: + +* project_name: The name of the OpenStack project to migrate. +* version: The version of the project. The default is "master". + e.g. stable-2025.1 +* workspace_name: The folder name for the migration workspace. +It will be installed in the home directory. The default is "workspace". + +Folder and File Structure +-------------- +.. code-block:: text + + ├── setup_env/ + └── migration_resources.sh + +* setup_env/: Scripts to create a virtual environment and install dependencies. +These scripts also create a workspace folder for migration tasks. +* migration_resources.sh: Main script to migrate translation resources. +They include various modules to perform actual tasks for resource +migration step by step. + +.. note:: + + The current tool mainly focuses on translation resource migration, + and actual resource migration process currently requires careful + supervision to make sure that each migration step is performed + well or not. + +Workspace Structure +------------------- + +By default, the migration workspace is installed to your home directory +with the following folder structure: + +.. note:: + + You can change the base directory by manually setting WORK_DIR + on setup_env/setup.sh. + +* .venv/: Python virtual environment containing migration dependencies +* projects/: Migration workspace + + * /: Project-specific workspace + + * /: Cloned project repository + * pot/: POT files for each component + * translations/: Exported translations from Zanata + +Directory Layout:: + +.. code-block:: text + + / + ├── .venv/ + └── projects/ + └── / + ├── / + ├── pot/ + └── translations/ diff --git a/tools/migration/migration_resources.sh b/tools/migration/migration_resources.sh new file mode 100755 index 0000000..51a5d1d --- /dev/null +++ b/tools/migration/migration_resources.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Migration script to create Weblate components and translations + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +PROJECT=$1 +BRANCH_NAME=${2:-"master"} +WORKSPACE_NAME=${3:-"workspace"} + +# Replace /'s in branch names with -'s because Zanata doesn't +# allow /'s in version names. +ZANATA_VERSION=${BRANCH_NAME//\//-} + +# List the components to be handled +COMPONENTS=() + +SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $SCRIPTSDIR/setup_env/setup.sh + +# We need a UTF-8 locale, set it properly in case it's not set. +export LANG=en_US.UTF-8 + +# You should set WEBLATE_URL and WEBLATE_TOKEN +# system environment variables. +echo "[INFO] Check variables" +if [ -z "$WEBLATE_URL" ] || [ "$WEBLATE_URL" == "" ]; then + echo "[ERROR] WEBLATE_URL is not set" + exit 1 +fi +if [ -z "$WEBLATE_TOKEN" ] || [ "$WEBLATE_TOKEN" == "" ]; then + echo "[ERROR] WEBLATE_TOKEN is not set" + exit 1 +fi +echo "[INFO] WEBLATE_URL and WEBLATE_TOKEN are set" + +echo "[INFO] Setup environment and prepare workspace" +if ! setup_env_and_prepare_workspace "$PROJECT"; then + echo "[ERROR] Failed to setup environment and prepare workspace" + exit 1 +fi diff --git a/tools/migration/setup_env/bindep.txt b/tools/migration/setup_env/bindep.txt new file mode 100644 index 0000000..af07bcf --- /dev/null +++ b/tools/migration/setup_env/bindep.txt @@ -0,0 +1,5 @@ +# This is a cross-platform list tracking distribution packages needed by tests; +# see https://docs.openstack.org/infra/bindep/ for additional information. + +gettext +jq \ No newline at end of file diff --git a/tools/migration/setup_env/requirements.txt b/tools/migration/setup_env/requirements.txt new file mode 100644 index 0000000..8d4b63b --- /dev/null +++ b/tools/migration/setup_env/requirements.txt @@ -0,0 +1,3 @@ +bindep +polib==1.2.0 +lxml==5.3.0 diff --git a/tools/migration/setup_env/setup.sh b/tools/migration/setup_env/setup.sh new file mode 100644 index 0000000..6102a14 --- /dev/null +++ b/tools/migration/setup_env/setup.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# Set venv and folder structure for migration + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +WORK_DIR="$HOME/$WORKSPACE_NAME" + +function create_python_venv() { + if ! command -v python3 &> /dev/null; then + echo "[ERROR] Python 3 is not installed" + return 1 + fi + + # create venv + if [ ! -d "$WORK_DIR/.venv" ]; then + python3 -m venv "$WORK_DIR/.venv" >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to create virtual environment" + return 1 + fi + fi + + echo "[INFO] Virtual environment created" + return 0 +} + +function install_dependencies() { + source "$WORK_DIR/.venv/bin/activate" + + cd "$SCRIPTSDIR/setup_env" + # Install python dependencies + pip3 install -r requirements.txt + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to install requirements.txt in venv." + return 1 + fi + + # Install dependencies with bindep + bindep_packages=$("$WORK_DIR/.venv/bin/bindep" -b -f bindep.txt 2>/dev/null) + if [ -n "$bindep_packages" ]; then + echo "[INFO] Installing system dependencies with bindep: $bindep_packages" + sudo apt install -y $bindep_packages + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to install system dependencies with bindep" + return 1 + fi + fi + + return 0 +} + +function check_zanata_cli() { + # Check zanata-cli is installed + if ! command -v zanata-cli &> /dev/null; then + echo "[ERROR] zanata-cli is not installed" + return 1 + fi + + # Check zanata.ini file exists + if [ ! -f "$HOME/.config/zanata.ini" ]; then + echo "[ERROR] zanata.ini is not found" + return 1 + fi + echo "[INFO] zanata-cli is installed and zanata.ini file exists" + return 0 +} + +function setup_env() { + if ! prepare_workspace; then + echo "[ERROR] Failed to prepare workspace." + return 1 + fi + + if ! create_python_venv; then + echo "[ERROR] Failed to create python venv" + return 1 + fi + + if ! install_dependencies; then + echo "[ERROR] Failed to install dependencies." + echo "[ERROR] Please check bindep.txt and requirements.txt" + return 1 + fi + + if ! check_zanata_cli; then + echo "[ERROR] Failed to check zanata-cli." + echo "[ERROR] Please check if zanata-cli is installed and zanata.ini file exists." + return 1 + fi + + echo "[INFO] Environment setup successfully" + return 0 +} + +function prepare_workspace() { + # Create workspace directory + if [ ! -d "$WORK_DIR" ]; then + if ! mkdir -p "$WORK_DIR"; then + echo "[ERROR] Failed to create $WORKSPACE_NAME directory" + return 1 + fi + fi + + # Create projects directory + if [ ! -d "$WORK_DIR/projects" ]; then + if ! mkdir -p "$WORK_DIR/projects"; then + echo "[ERROR] Failed to create projects directory" + return 1 + fi + fi + + echo "[INFO] Workspace directory created successfully" + return 0 + +} + +function prepare_project_workspace() { + local project=$1 + + # Create project directory + if [ ! -d "$WORK_DIR/projects/$project" ]; then + mkdir -p $WORK_DIR/projects/$project + fi + + # Create pot directory + if [ ! -d "$WORK_DIR/projects/$project/pot" ]; then + mkdir -p $WORK_DIR/projects/$project/pot + echo "[INFO] Pot directory created successfully" + fi + + # Create translations directory + if [ ! -d "$WORK_DIR/projects/$project/translations" ]; then + mkdir -p $WORK_DIR/projects/$project/translations + echo "[INFO] Translations directory created successfully" + fi + + return 0 +} + +function setup_env_and_prepare_workspace() { + local project=$1 + + # Create a virtual environment and install system dependencies + echo "[INFO] Setup the virtual environment and install system dependencies" + if ! setup_env; then + echo "[ERROR] Failed to setup environment" + return 1 + fi + + # Prepare workspace folders for migration tasks + echo "[INFO] Prepare workspace folders" + if ! prepare_workspace; then + echo "[ERROR] Failed to prepare workspace" + return 1 + fi + + prepare_project_workspace "$project" + return 0 +} \ No newline at end of file diff --git a/tools/migration/tox.ini b/tools/migration/tox.ini new file mode 100644 index 0000000..26eef51 --- /dev/null +++ b/tools/migration/tox.ini @@ -0,0 +1,9 @@ +[tox] +minversion = 3.1.0 +envlist= py{39,310,311,312,313} +skipdist = True + +[testenv] +deps = -r {toxinidir}/setup_env/requirements.txt +commands = + pip check \ No newline at end of file