From dd43fa560a7ab1bbb9467a0bb12594270e8c6db4 Mon Sep 17 00:00:00 2001 From: Takashi Kajinami Date: Mon, 14 Oct 2024 20:58:20 +0900 Subject: [PATCH] Drop db migration tool Placement was split off from nova during Stein release and we no longer expect any users may use this tool from master. Change-Id: I1283980ae8640b2da4bbeae1856c3df821dc4975 --- doc/source/_extra/.htaccess | 1 - doc/source/admin/index.rst | 2 - doc/source/admin/upgrade-notes.rst | 62 ---- doc/source/admin/upgrade-to-stein.rst | 223 -------------- doc/source/install/index.rst | 20 -- doc/test/redirect-tests.txt | 1 - placement_db_tools/__init__.py | 21 -- placement_db_tools/mysql-migrate-db.sh | 307 -------------------- placement_db_tools/postgresql-migrate-db.sh | 287 ------------------ setup.cfg | 1 - 10 files changed, 925 deletions(-) delete mode 100644 doc/source/admin/upgrade-to-stein.rst delete mode 100644 placement_db_tools/__init__.py delete mode 100755 placement_db_tools/mysql-migrate-db.sh delete mode 100755 placement_db_tools/postgresql-migrate-db.sh diff --git a/doc/source/_extra/.htaccess b/doc/source/_extra/.htaccess index 1befc5f39..3d2d96d40 100644 --- a/doc/source/_extra/.htaccess +++ b/doc/source/_extra/.htaccess @@ -1,6 +1,5 @@ redirectmatch 301 ^/placement/([^/]+)/specs/train/approved/2005297-negative-aggregate-membership.html /placement/$1/specs/train/implemented/2005297-negative-aggregate-membership.html redirectmatch 301 ^/placement/([^/]+)/specs/train/approved/placement-resource-provider-request-group-mapping-in-allocation-candidates.html /placement/$1/specs/train/implemented/placement-resource-provider-request-group-mapping-in-allocation-candidates.html redirectmatch 301 ^/placement/([^/]+)/specs/train/approved/2005575-nested-magic-1.html /placement/$1/specs/train/implemented/2005575-nested-magic-1.html -redirectmatch 301 ^/placement/([^/]+)/upgrade/to-stein.html /placement/$1/admin/upgrade-to-stein.html redirectmatch 301 ^/placement/([^/]+)/usage/index.html /placement/$1/user/index.html redirectmatch 301 ^/placement/([^/]+)/usage/provider-tree.html /placement/$1/user/provider-tree.html diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index 5235589fc..98725d503 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -18,5 +18,3 @@ Upgrade :maxdepth: 2 upgrade-notes - upgrade-to-stein - diff --git a/doc/source/admin/upgrade-notes.rst b/doc/source/admin/upgrade-notes.rst index 07c1f9ecb..0996b70d0 100644 --- a/doc/source/admin/upgrade-notes.rst +++ b/doc/source/admin/upgrade-notes.rst @@ -23,65 +23,3 @@ This section provide notes on upgrading to a given target release. :ref:`placement-status upgrade check ` tool can be used to help determine the status of your deployment and how ready it is to perform an upgrade. - -For releases prior to Stein, please see the `nova upgrade notes`_. - -.. _nova upgrade notes: https://docs.openstack.org/nova/rocky/user/placement.html#upgrade-notes - -Train (2.0.0) -~~~~~~~~~~~~~ - -The Train release of placement is the first release where placement is -available solely from its own project and must be installed separately from -nova. If the extracted placement is not already in use, prior to upgrading to -Train, the Stein version of placement must be installed. See the next section -and :doc:`upgrade-to-stein` for details. - -There are no database schema changes in the Train release, but there are -checks to confirm that online migrations from Stein have been run. Running -:doc:`/cli/placement-status` *after upgrading code but prior to restarting the -placement service* will notify you of any missing steps and the process to fix -it. Once this is done, :doc:`/cli/placement-manage` should be run to sync the -database:: - - $ placement-status upgrade check - +----------------------------------+ - | Upgrade Check Results | - +----------------------------------+ - | Check: Missing Root Provider IDs | - | Result: Success | - | Details: None | - +----------------------------------+ - | Check: Incomplete Consumers | - | Result: Success | - | Details: None | - +----------------------------------+ - $ placement-manage db sync - -Then the placement service may be restarted. - -Stein (1.0.0) -~~~~~~~~~~~~~ - -If you are upgrading an existing OpenStack installation from Rocky to Stein, -and wish to use the newly extracted placement, you will need to copy some -data and configuration settings from nova. - -* Configuration and policy files are, by default, located in - ``/etc/placement``. -* The placement server side settings in ``nova.conf`` should be moved to a - separate placement configuration file ``placement.conf``. -* The default configuration value of ``[placement]/policy_file`` is changed - from ``placement-policy.yaml`` to ``policy.yaml``. This config option is - changed to :oslo.config:option:`oslo_policy.policy_file` since Train - release. -* Several tables in the ``nova_api`` database need to be migrated to a new - ``placement`` database. - -Following these steps will ensure that future changes to placement -configuration and code will not conflict with your setup. - -As stated above, using the extracted placement code is not required in Stein, -there is a copy in the Stein release of Nova. However that code will be deleted -in the Train cycle so you must upgrade to external Placement prior to -upgrading to Train. diff --git a/doc/source/admin/upgrade-to-stein.rst b/doc/source/admin/upgrade-to-stein.rst deleted file mode 100644 index 75bc1c4dc..000000000 --- a/doc/source/admin/upgrade-to-stein.rst +++ /dev/null @@ -1,223 +0,0 @@ -.. - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -================================ -Upgrading from Nova to Placement -================================ - -This document is for people who are upgrading from an existing Rocky-based -installation of OpenStack, where Placement is a part of Nova, to a Stein-based -system, using the independently packaged placement service. It is also for -people who have already upgraded to Stein but are using the version of the -placement service included in Nova in the Stein release. - -Upgrading to the extracted placement is not a requirement when upgrading the -rest of OpenStack to Stein. The version of the placement service in the -Nova Stein release may be used. It is possible to upgrade to Stein and then -deploy and switch to the extracted placement at a later time. - -The placement code in Nova will be removed in Train so the switch to using -extracted placement must happen before upgrading to Train. - -.. note:: The extracted placement code has features and performance and bug - fixes that are not present in the placement code in Nova, but no code - that is required by Nova. See the `release notes`_ for more detail. - -If you are installing a new OpenStack, you will want the -:doc:`installation docs `. - -Upgrading to use the extracted placement service requires migrating several -database tables from the ``nova_api`` database to a placement database. -Depending on the number of compute hosts in your system and the number of -active virtual machines, the amount of data to copy can vary widely. You can -get an idea by counting rows in the ``resource_providers`` and ``consumers`` -tables. - -To avoid losing data while performing the copy it is important that writing to -the placement database (on either side of the upgrade) is stopped. You may shut -down solely the placement service but this will result in errors attempting to -use the service from Nova. It is potentially less disruptive to shut down the -entire control plane to avoid confusing errors. What strategy is best will -vary. This document describes the simple way. - -.. note:: In some installations of nova and placement, data may already be in - a database named ``placement`` and not ``nova_api``. If that is the - case, you will not need to copy data. Make sure that there are tables - and rows in that database and that it is of expected quantity and - recently modified (many tables have ``created_at`` and ``updated_at`` - columns). In some cases the ``placement`` database will be present - *but empty*. - -There are database migrations scripts in the placement code repository which -may be used to copy the data or as models for your own tooling: -`mysql-migrate-db.sh`_ and `postgresql-migrate-db.sh`_. - -.. note:: Starting in the Train release, these migration scripts are also - packaged with the `openstack-placement`_ package on PyPI. Their - filenames may be discovered using ``pkg_resources`` to look in the - ``placement_db_tools`` package:: - - pkg_resources.resource_filename('placement_db_tools', 'mysql-migrate-db.sh') - -For best results run the database migration on your database host. If you are -unable to do this, you will need to take some additional steps below. - -This document assumes that the same HTTP endpoint will be used before and after -the upgrade. If you need to change that see :ref:`configure-endpoints-pypi` for -guidance. - -Initial Steps -------------- - -#. Install the new placement code on a controller node. This can be - `openstack-placement`_ from PyPI or you can use packages from a Linux - distribution. If you are using the latter be aware that: - - * The name of the package can be found in the :doc:`installation docs - `. - - * You need to install the packages on a different host from the old nova, - to avoid accidentally upgrading before you are ready. - -#. Create a ``placement`` database with appropriate access controls. If you - need details on how to do this, see :ref:`create-database-pypi`. - -#. Create and configure the ``placement.conf`` file. - - * The default location is ``/etc/placement``. - - * Set :oslo.config:option:`placement_database.connection` to point to the - new database. For example (replacing ``PLACEMENT_DBPASS`` and - ``controller`` with the appropriate password and host): - - .. path /etc/placement/placement.conf - .. code-block:: ini - - [placement_database] - connection = mysql+pymysql://placement:PLACEMENT_DBPASS@controller/placement - - * Configure the :oslo.config:group:`keystone_authtoken` section as described - in :ref:`configure-conf-pypi`. - - * If the following configuration settings are set in the ``[placement]`` - section of ``/etc/nova/nova.conf``, move them to a ``[placement]`` - section in ``/etc/placement/placement.conf``: - - * :oslo.config:option:`placement.randomize_allocation_candidates` - * :oslo.config:option:`placement.incomplete_consumer_project_id` - * :oslo.config:option:`placement.incomplete_consumer_user_id` - -#. Move ``placement-policy.yaml``, if required. - - * If it exists, move ``/etc/nova/placement-policy.yaml`` to - ``/etc/placement/policy.yaml``. If you wish to use a different filename - adjust config option ``[placement] policy_file``. - -#. Configure the database migration tool. - - * Create the configuration file. - - .. note:: The examples in this guide are using MySQL but if you are using - PostgreSQL it is recommended to use the - `postgresql-migrate-db.sh`_ script since it handles sequences. - See `bug 2005478`_ for details. - - .. code-block:: console - - $ mysql-migrate-db.sh --mkconfig /root/migrate-db.rc - - * Edit the file to set the values for the ``NOVA_API_USER``, - ``NOVA_API_PASS``, ``PLACEMENT_USER``, and ``PLACEMENT_PASS`` entries. - These are the usernames and passwords for accessing the database. - - * If you are unable to run the migration script on the database host you - will need to set ``NOVA_API_DB_HOST`` and ``PLACEMENT_DB_HOST``. - - * Do not change ``MIGRATE_TABLES`` unless you need to migrate tables - incrementally. - -#. Configure the web server that will host the placement service. The details - of this are beyond the scope of this document. :doc:`/install/from-pypi` - may provide some guidance. **Make sure you also disable the previously - running placement service in the web server configuration.** - -Migrate the Data ----------------- - -#. Shut down or disable your control plane in whatever way works best for you. - -#. Run the migration script: - - .. code-block:: console - - $ mysql-migrate-db.sh --migrate /root/migrate-db.rc - - The ``--skip-locks`` flag can be used along with ``--migrate`` in - deployments where table locking operations can't be performed. For example, - Percona XtraDB Cluster only has experimental support for explicit table - locking operations and attempts to use locking will result in errors when - PXC Strict Mode is set to ENFORCING. - - If your controller host (the one where you have been editing - ``/etc/placement/placement.conf``) and database host are not the same, and - you have run the migration script on the database host, the final step in - the process will fail. This step stamps the database with an initial version - (the hash of the first alembic_ migration) so that future migrations will - work properly. From the controller host, you may do it manually with: - - .. code-block:: console - - $ placement-manage db stamp b4ed3a175331 - -#. Sync the placement database to be up to date with all migrations: - - .. code-block:: console - - $ placement-manage db sync - -.. note:: - - As described in `bug 1978549`_ the ``can_host`` column of the - ``resource_providers`` table was removed from the DB model but not from the - DB schema while Placement was still part of Nova. Then when Placement was - split out its DB schema was altered to not contain ``can_host`` any - more. This can create a situation when the actual DB schema and the schema - defined by the alembic code is different. As ``can_host`` is not used any - more it is safe to manually remove it from the DB to remove the schema - inconsistency. - -Finalize the Upgrade --------------------- - -#. Start up the new placement service. - -#. Restart your control plane services. If you are upgrading to Stein, continue - with the upgrade of the rest of the system. - -#. Verify the content of the new service by using the osc-placement_ tool to - list resource providers, allocations and other resources in the service. - -#. Verify the integration of placement with the rest of your OpenStack - installation by creating and deleting a test server. - -#. At some point in the future you may remove the tables in the ``nova_api`` - database that were migrated to the ``placement`` database. - -.. _openstack-placement: https://pypi.org/p/openstack-placement -.. _mysql-migrate-db.sh: https://opendev.org/openstack/placement/raw/branch/master/placement_db_tools/mysql-migrate-db.sh -.. _postgresql-migrate-db.sh: https://opendev.org/openstack/placement/raw/branch/master/placement_db_tools/postgresql-migrate-db.sh -.. _alembic: https://alembic.sqlalchemy.org/en/latest/ -.. _release notes: https://docs.openstack.org/releasenotes/placement/stein.html -.. _osc-placement: https://docs.openstack.org/osc-placement/latest/ -.. _bug 2005478: https://storyboard.openstack.org/#!/story/2005478 -.. _bug 1978549: https://bugs.launchpad.net/nova/+bug/1978549 diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst index c6db730e8..504bf217e 100644 --- a/doc/source/install/index.rst +++ b/doc/source/install/index.rst @@ -90,26 +90,6 @@ placement web service starts. Whether you choose to sync automaticaly or use the command line tool depends on the constraints of your environment and deployment tooling. -.. warning:: In the Stein release, the placement code was extracted from nova. - If you are upgrading to use the extracted placement you will need - to migrate your placement data from the ``nova_api`` database to - the ``placement`` database. You can find sample scripts that may - help with this in the `placement repository`_: - `mysql-migrate-db.sh`_ and `postgresql-migrate-db.sh`_. - See also :doc:`/admin/upgrade-notes`. - -.. note:: Upgrading to the extracted placement at the same time as the other - OpenStack services when upgrading to Stein is an option but *is not - required*. The nova code will continue to have a copy of the - placement service in its Stein release. However this copy **will** be - deleted in Train and switching to the extracted version before - upgrading to Train (potentially with the help of the scripts above) - will be required. - -.. _placement repository: https://opendev.org/openstack/placement -.. _mysql-migrate-db.sh: https://opendev.org/openstack/placement/raw/branch/master/tools/mysql-migrate-db.sh -.. _postgresql-migrate-db.sh: https://opendev.org/openstack/placement/raw/branch/master/tools/postgresql-migrate-db.sh - **3. Create accounts and update the service catalog** Create a **placement** service user with an **admin** role in Keystone. diff --git a/doc/test/redirect-tests.txt b/doc/test/redirect-tests.txt index ba3d4e58d..7a0514dd1 100644 --- a/doc/test/redirect-tests.txt +++ b/doc/test/redirect-tests.txt @@ -1,6 +1,5 @@ /placement/latest/specs/train/approved/2005297-negative-aggregate-membership.html 301 /placement/latest/specs/train/implemented/2005297-negative-aggregate-membership.html /placement/latest/specs/train/approved/placement-resource-provider-request-group-mapping-in-allocation-candidates.html 301 /placement/latest/specs/train/implemented/placement-resource-provider-request-group-mapping-in-allocation-candidates.html /placement/latest/specs/train/approved/2005575-nested-magic-1.html 301 /placement/latest/specs/train/implemented/2005575-nested-magic-1.html -/placement/latest/upgrade/to-stein.html 301 /placement/latest/admin/upgrade-to-stein.html /placement/latest/usage/index.html 301 /placement/latest/user/index.html /placement/latest/usage/provider-tree.html 301 /placement/latest/user/provider-tree.html diff --git a/placement_db_tools/__init__.py b/placement_db_tools/__init__.py deleted file mode 100644 index 01363eedb..000000000 --- a/placement_db_tools/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -This package provides a way to distribute useful database management tools as -part of the Python package of the placement service: openstack-placement. - -Their filenames may be discovered using ``pkg_resources`` to look in the -``placement_db_tools`` package: - - pkg_resources.resource_filename('placement_db_tools', - 'mysql-migrate-db.sh') -""" diff --git a/placement_db_tools/mysql-migrate-db.sh b/placement_db_tools/mysql-migrate-db.sh deleted file mode 100755 index 2a9f32dc6..000000000 --- a/placement_db_tools/mysql-migrate-db.sh +++ /dev/null @@ -1,307 +0,0 @@ -#!/bin/bash - -# This script will attempt to migrate your nova-api placement data to -# a new placement database. Run it with --help for usage, and --mkconfig -# to write a template config file to use. - -# Defaults we can guess -DEFAULT_MIGRATE_TABLES="allocations placement_aggregates consumers inventories projects " -DEFAULT_MIGRATE_TABLES+="resource_classes resource_provider_aggregates resource_provider_traits " -DEFAULT_MIGRATE_TABLES+="resource_providers traits users " -MIGRATE_TABLES=${MIGRATE_TABLES:-$DEFAULT_MIGRATE_TABLES} -PLACEMENT_DB_HOST=${PLACEMENT_DB_HOST:-localhost} -PLACEMENT_DB=${PLACEMENT_DB:-placement} -NOVA_API_DB_HOST=${NOVA_API_DB_HOST:-localhost} -NOVA_API_DB=${NOVA_API_DB:-nova_api} -TMPDIR=${TMPDIR:-/tmp} -LAST_MYSQL_ERR=${TMPDIR}/migrate-mysql-db.err -INITIAL_PLACEMENT_DB_VERSION=${INITIAL_DB_VERSION:-b4ed3a175331} -ME=$(basename "$0") - -declare -a ARGS -declare -a OPTS - -function getflag() { - # Return true if --$flag is present on the command line - # Usage: getflag help -> 0 - local flag="$1" - for opt in ${OPTS[*]}; do - if [ "$opt" == "--${flag}" ]; then - return 0 - fi - done - return 1 -} - -function parse_argv() { - # Parse command line arguments into positional arguments and - # option flags. Store each in $ARGS, $OPTS. - # Usage: parse_argv $* - for item in $*; do - if echo $item | grep -q -- '^--'; then - OPTS+=($item) - else - ARGS+=($item) - fi - done -} - -function db_var() { - # Return an attribute of database config based on the symbolic - # name - # Usage: db_var PLACEMENT USER -> $PLACEMENT_USER - local db="$1" - local var="$2" - - eval echo "\$${db}_${var}" -} - -function mysql_command() { - # Run a mysql command with the usual connection information taken - # from a symbolic configuration name - # Usage: mysql_command PLACEMENT [command] [args..] -> stdout - local whichdb="$1" - shift - local command=mysql - if [ "$2" ]; then - command=${1:-mysql} - shift - fi - local db=$(db_var $whichdb DB) - local host=$(db_var $whichdb DB_HOST) - local user=$(db_var $whichdb USER) - local pass=$(db_var $whichdb PASS) - - if [ "$command" = "mysql" ]; then - command="mysql --skip-column-names" - fi - - $command -h$host -u$user -p$pass $db $* 2>$LAST_MYSQL_ERR -} - -function show_error() { - # Prints the last error (if present) and removes the temporary - # file - if [ -f $LAST_MYSQL_ERR ]; then - cat $LAST_MYSQL_ERR - rm -f $LAST_MYSQL_ERR - fi -} - -function check_db() { - # Check a DB to see if it's missing, present, filled with data - # Returns 0 if it is present with data, 1 if present but no data - # or 2 if not present (or unable to connect) - # Usage: check_db PLACEMENT -> 0 - local whichdb="$1" - - local inv - local inv_count - - if ! echo "SELECT DATABASE()" | mysql_command $whichdb >/dev/null 2>&1; then - echo "Failed to connect to $whichdb database" - show_error - return 2 - fi - - inv=$(echo "SELECT COUNT(id) FROM inventories" | - mysql_command $whichdb) - if [ $? -ne 0 ]; then - # No schema - return 1 - fi - - inv_count=$(echo $inv | tail -n1) - if [ $inv_count -gt 0 ]; then - # Data found - return 0 - else - # No data found, but schema intact - return 1 - fi -} - -function check_cli() { - # Returns 0 if placement cli is installed and configured, - # 1 if it is not installed, or 2 if the access to the - # placement database fails - # Usage: check_cli -> 0 - placement-manage --version > /dev/null 2>&1 - - if [ $? -ne 0 ]; then - # placement not installed - return 1 - fi - - placement-manage db version > /dev/null 2>&1 - - if [ $? -ne 0 ]; then - # DB connection fails - return 2 - fi -} - -function migrate_data() { - # Actually migrate data from a source to destination symbolic - # database. Returns 1 if failure, 0 otherwise. - # Usage: migrate_data NOVA_API PLACEMENT -> 0 - local source="$1" - local dest="$2" - local dump_flags="$3" - local tmpdir=$(mktemp -d migrate-db.XXXXXXXX) - local tmpfile="${tmpdir}/from-nova.sql" - - echo "Dumping from $source to $tmpfile" - mysql_command $source mysqldump $dump_flags $MIGRATE_TABLES > $tmpfile || { - echo 'Failed to dump source database:' - show_error - return 1 - } - echo "Loading to $dest from $tmpfile" - mysql_command $dest < $tmpfile || { - echo 'Failed to load destination database:' - show_error - return 1 - } -} - -function sanity_check_env() { - # Check that we have everything we need to examine the situation - # and potentially do the migration. Loads values from the rcfile, - # if present. Returns 1 if a config was not found, 2 if that - # config is incomplete or 0 if everything is good. - # Usage: sanity_check_env $rcfile -> 0 - - RCFILE="${1:-migrate-db.rc}" - if [ "$RCFILE" = '-' ]; then - # Don't require a file and assume everything is already - # set in the environment - true - elif [ ! -f "$RCFILE" ]; then - echo -n 'ERROR: Specify an RC file on the command line or create ' - echo 'migrate-db.rc in the current directory' - echo - show_help - else - source $RCFILE - fi - - required="NOVA_API_DB NOVA_API_USER NOVA_API_PASS PLACEMENT_DB PLACEMENT_USER PLACEMENT_PASS" - for var in $required; do - value=$(eval echo "\$$var") - if [ -z "$value" ]; then - echo "A value for $var was not provided but is required" - return 2 - fi - done -} - -function make_config() { - # Create or update a config file with defaults we know. Either use - # the default migrate-db.rc or the file specified on the command - # line. - RCFILE="${1:-migrate-db.rc}" - if [ -f "$RCFILE" ]; then - source $RCFILE - fi - - vars="NOVA_API_DB NOVA_API_USER NOVA_API_PASS NOVA_API_DB_HOST " - vars+="PLACEMENT_DB PLACEMENT_USER PLACEMENT_PASS PLACEMENT_DB_HOST " - vars+="MIGRATE_TABLES" - - (for var in $vars; do - val=$(eval echo "\$$var") - echo "${var}=\"$val\"" - done) > $RCFILE - - echo Wrote $(readlink -f $RCFILE) -} - -function show_help() { - echo "Usage: $ME [flags] [rcfile]" - echo - echo "Flags:" - echo " --help: this text" - echo " --migrate: actually do data migration" - echo " --mkconfig: write/update config to \$rcfile" - echo " --skip-locks: don't use table locks for data migration" - echo - echo "Pass '-' as \$rcfile if all config values are set in" - echo "the environment." - echo - echo "Exit codes:" - echo " 0: Success" - echo " 1: Usage error" - echo " 2: Configuration missing or incomplete" - echo " 3: Migration already completed" - echo " 4: No data to migrate from nova (new deployment)" - echo " 5: Unable to connect to one or both databases" - echo " 6: Unable to execute placement's CLI commands" - exit 0 -} - -parse_argv $* - -if getflag help; then - show_help -fi - -if getflag mkconfig; then - make_config $ARGS - exit 0 -fi - -# -# Actual migration logic starts here -# - -# Sanity check that we have what we need or bail -sanity_check_env $ARGS || exit $? - -# Check the state of each database we care about -check_db NOVA_API -nova_present=$? -check_db PLACEMENT -placement_present=$? -check_cli -placement_cli=$? - -# Try to come up with a good reason to refuse to migrate -if [ $nova_present -eq 0 -a $placement_present -eq 0 ]; then - echo "Migration has already completed. The placement database appears to have data." - exit 3 -elif [ $nova_present -eq 1 ]; then - echo "No data present in nova database - nothing to migrate (new deployment?)" - exit 4 -elif [ $nova_present -eq 2 ]; then - echo "Unable to proceed without connection to nova database" - exit 5 -elif [ $placement_present -eq 2 ]; then - echo "Unable to proceed without connection to placement database" - exit 5 -elif [ $placement_cli -eq 1 ]; then - echo "Unable to proceed without placement installed" - exit 6 -elif [ $placement_cli -eq 2 ]; then - echo "The 'placement-manage db version' command fails" - echo "Is placement.conf configured to access the new database?" - exit 6 -fi - -# If we get here, we expect to be able to migrate. Require them to opt into -# actual migration before we do anything. - -echo Nova database contains data, placement database does not. Okay to proceed with migration - -if getflag migrate $*; then - if getflag skip-locks $*; then - migrate_data NOVA_API PLACEMENT "--skip-lock-tables --skip-add-locks" - else - migrate_data NOVA_API PLACEMENT - fi - placement-manage db stamp $INITIAL_PLACEMENT_DB_VERSION -else - echo "To actually migrate, run me with --migrate" -fi - -rm -f $LAST_MYSQL_ERR diff --git a/placement_db_tools/postgresql-migrate-db.sh b/placement_db_tools/postgresql-migrate-db.sh deleted file mode 100755 index 9bd6732cf..000000000 --- a/placement_db_tools/postgresql-migrate-db.sh +++ /dev/null @@ -1,287 +0,0 @@ -#!/bin/bash - -# This script will attempt to migrate your nova-api placement data to -# a new placement database. Run it with --help for usage, and --mkconfig -# to write a template config file to use. - -# Defaults we can guess -DEFAULT_MIGRATE_TABLES="allocations placement_aggregates consumers inventories projects " -DEFAULT_MIGRATE_TABLES+="resource_classes resource_provider_aggregates resource_provider_traits " -DEFAULT_MIGRATE_TABLES+="resource_providers traits users " -MIGRATE_TABLES=${MIGRATE_TABLES:-$DEFAULT_MIGRATE_TABLES} -PG_MIGRATE_TABLES=${MIGRATE_TABLES// /|}} -PLACEMENT_DB_HOST=${PLACEMENT_DB_HOST:-localhost} -PLACEMENT_DB=${PLACEMENT_DB:-placement} -NOVA_API_DB_HOST=${NOVA_API_DB_HOST:-localhost} -NOVA_API_DB=${NOVA_API_DB:-nova_api} -TMPDIR=${TMPDIR:-/tmp} -LAST_PSQL_ERR=${TMPDIR}/migrate-psql-db.err -INITIAL_PLACEMENT_DB_VERSION=${INITIAL_DB_VERSION:-b4ed3a175331} - -declare -a ARGS -declare -a OPTS - -function getflag() { - # Return true if --$flag is present on the command line - # Usage: getflag help -> 0 - local flag="$1" - for opt in ${OPTS[*]}; do - if [ "$opt" == "--${flag}" ]; then - return 0 - fi - done - return 1 -} - -function parse_argv() { - # Parse command line arguments into positional arguments and - # option flags. Store each in $ARGS, $OPTS. - # Usage: parse_argv $* - for item in $*; do - if echo $item | grep -q -- '^--'; then - OPTS+=($item) - else - ARGS+=($item) - fi - done -} - -function db_var() { - # Return an attribute of database config based on the symbolic - # name - # Usage: db_var PLACEMENT USER -> $PLACEMENT_USER - local db="$1" - local var="$2" - - eval echo "\$${db}_${var}" -} - -function psql_command() { - # Run a psql command with the usual connection information taken - # from a symbolic configuration name - # Usage: psql_command PLACEMENT [command] [args..] -> stdout - local whichdb="$1" - shift - local command=psql - if [ "$2" ]; then - command=${1:-psql} - shift - fi - local db=$(db_var $whichdb DB) - local host=$(db_var $whichdb DB_HOST) - local user=$(db_var $whichdb USER) - local pass=$(db_var $whichdb PASS) - - if [ "$command" = "psql" ]; then - command="psql -t" - fi - - PGPASSWORD=$pass $command -h$host -U$user $db $* 2>$LAST_PSQL_ERR -} - -function check_db() { - # Check a DB to see if it's missing, present, filled with data - # Returns 0 if it is present with data, 1 if present but no data - # or 2 if not present (or unable to connect) - # Usage: check_db PLACEMENT -> 0 - local whichdb="$1" - - local inv - local inv_count - local error_found - - if ! echo "SELECT CURRENT_DATABASE()" | psql_command $whichdb >/dev/null 2>&1; then - echo "Failed to connect to $whichdb database" - return 2 - fi - - inv=$(echo "SELECT COUNT(id) FROM inventories" | - psql_command $whichdb) - if [ $? -ne 0 ]; then - # No DB - return 1 - fi - - error_found=$(cat $LAST_PSQL_ERR | grep ERROR) - if [ $? -eq 0 ]; then - # No schema - return 1 - fi - - inv_count=$(echo $inv | tail -n1) - if [ $inv_count -gt 0 ]; then - # Data found - return 0 - else - # No data found, but schema intact - return 1 - fi -} - -function check_cli() { - # Returns 0 if placement cli is installed and configured, - # 1 if it is not installed, or 2 if the access to the - # placement database fails - # Usage: check_cli -> 0 - placement-manage --version > /dev/null 2>&1 - - if [ $? -ne 0 ]; then - # placement not installed - return 1 - fi - - placement-manage db version > /dev/null 2>&1 - - if [ $? -ne 0 ]; then - # DB connection fails - return 2 - fi -} - -function migrate_data() { - # Actually migrate data from a source to destination symbolic - # database. Returns 1 if failure, 0 otherwise. - # Usage: migrate_data NOVA_API PLACEMENT -> 0 - local source="$1" - local dest="$2" - local tmpdir=$(mktemp -d migrate-db.XXXXXXXX) - local tmpfile="${tmpdir}/from-nova.sql" - - echo "Dumping from $source to $tmpfile" - psql_command $source pg_dump -t $PG_MIGRATE_TABLES > $tmpfile || { - echo 'Failed to dump source database:' - cat $LAST_PSQL_ERR - return 1 - } - echo "Loading to $dest from $tmpfile" - # There is some output when loading file, so we redirect it to /dev/null. - psql_command $dest < $tmpfile >/dev/null || { - echo 'Failed to load destination database:' - cat $LAST_PSQL_ERR - return 1 - } -} - -function sanity_check_env() { - # Check that we have everything we need to examine the situation - # and potentially do the migration. Loads values from the rcfile, - # if present. Returns 1 if a config was not found, 2 if that - # config is incomplete or 0 if everything is good. - # Usage: sanity_check_env $rcfile -> 0 - - RCFILE="${1:-migrate-db.rc}" - if [ ! -f "$RCFILE" ]; then - echo 'Specify an RC file on the command line or create migrate-db.rc in the current directory' - return 1 - fi - - source $RCFILE - - required="NOVA_API_DB NOVA_API_USER NOVA_API_PASS PLACEMENT_DB PLACEMENT_USER PLACEMENT_PASS" - for var in $required; do - value=$(eval echo "\$$var") - if [ -z "$value" ]; then - echo "A value for $var was not provided but is required" - return 2 - fi - done - -} - -function make_config() { - # Create or update a config file with defaults we know. Either use - # the default migrate-db.rc or the file specified on the command - # line. - RCFILE="${1:-migrate-db.rc}" - if [ -f "$RCFILE" ]; then - source $RCFILE - fi - - vars="NOVA_API_DB NOVA_API_USER NOVA_API_PASS NOVA_API_DB_HOST " - vars+="PLACEMENT_DB PLACEMENT_USER PLACEMENT_PASS PLACEMENT_DB_HOST " - vars+="MIGRATE_TABLES" - - (for var in $vars; do - val=$(eval echo "\$$var") - echo "${var}=\"$val\"" - done) > $RCFILE - - echo Wrote $(readlink -f $RCFILE) -} - -parse_argv $* - -if getflag help; then - echo "Usage: $0 [flags] [rcfile]" - echo - echo "Flags:" - echo " --help: this text" - echo " --migrate: actually do data migration" - echo " --mkconfig: write/update config to \$rcfile" - echo - echo "Exit codes:" - echo " 0: Success" - echo " 1: Usage error" - echo " 2: Configuration missing or incomplete" - echo " 3: Migration already completed" - echo " 4: No data to migrate from nova (new deployment)" - echo " 5: Unable to connect to one or both databases" - echo " 6: Unable to execute placement's CLI commands" - exit 0 -fi - -if getflag mkconfig; then - make_config $ARGS - exit 0 -fi - -# -# Actual migration logic starts here -# - -# Sanity check that we have what we need or bail -sanity_check_env $ARGS || exit $? - -# Check the state of each database we care about -check_db NOVA_API -nova_present=$? -check_db PLACEMENT -placement_present=$? -check_cli -placement_cli=$? - -# Try to come up with a good reason to refuse to migrate -if [ $nova_present -eq 0 -a $placement_present -eq 0 ]; then - echo "Migration has already completed. The placement database appears to have data." - exit 3 -elif [ $nova_present -eq 1 ]; then - echo "No data present in nova database - nothing to migrate (new deployment?)" - exit 4 -elif [ $nova_present -eq 2 ]; then - echo "Unable to proceed without connection to nova database" - exit 5 -elif [ $placement_present -eq 2 ]; then - echo "Unable to proceed without connection to placement database" - exit 5 -elif [ $placement_cli -eq 1 ]; then - echo "Unable to proceed without placement installed" - exit 6 -elif [ $placement_cli -eq 2 ]; then - echo "The 'placement-manage db version' command fails" - echo "Is placement.conf configured to access the new database?" - exit 6 -fi - -# If we get here, we expect to be able to migrate. Require them to opt into -# actual migration before we do anything. - -echo Nova database contains data, placement database does not. Okay to proceed with migration - -if getflag migrate $*; then - migrate_data NOVA_API PLACEMENT - placement-manage db stamp $INITIAL_PLACEMENT_DB_VERSION -else - echo "To actually migrate, run me with --migrate" -fi - -rm -f $LAST_PSQL_ERR diff --git a/setup.cfg b/setup.cfg index 0a6f9d57e..925491115 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,6 @@ classifier = [files] packages = placement - placement_db_tools [entry_points] oslo.config.opts =