From 6ab254adbad32b5fb2d0b3e8c0fa9db4028f7c79 Mon Sep 17 00:00:00 2001 From: Andrew Laski Date: Thu, 19 Feb 2015 15:34:27 -0500 Subject: [PATCH] Add second migrate_repo for cells v2 database migrations This sets up the framework for adding database migrations to run on the new api database. It splits migration commands so that they can run on the current 'main' database or the new 'api' database. bp: cells-v2-mapping Change-Id: I73dee6fd51b74bd670a6dffa07e875bf86891e97 --- nova/db/migration.py | 12 +- nova/db/sqlalchemy/api_migrations/__init__.py | 0 .../api_migrations/migrate_repo/README | 4 + .../api_migrations/migrate_repo/__init__.py | 0 .../api_migrations/migrate_repo/migrate.cfg | 20 +++ .../migrate_repo/versions/__init__.py | 0 nova/db/sqlalchemy/migration.py | 35 ++-- nova/tests/functional/db/__init__.py | 0 nova/tests/functional/db/api/__init__.py | 0 .../functional/db/api/test_migrations.py | 149 ++++++++++++++++++ 10 files changed, 199 insertions(+), 21 deletions(-) create mode 100644 nova/db/sqlalchemy/api_migrations/__init__.py create mode 100644 nova/db/sqlalchemy/api_migrations/migrate_repo/README create mode 100644 nova/db/sqlalchemy/api_migrations/migrate_repo/__init__.py create mode 100644 nova/db/sqlalchemy/api_migrations/migrate_repo/migrate.cfg create mode 100644 nova/db/sqlalchemy/api_migrations/migrate_repo/versions/__init__.py create mode 100644 nova/tests/functional/db/__init__.py create mode 100644 nova/tests/functional/db/api/__init__.py create mode 100644 nova/tests/functional/db/api/test_migrations.py diff --git a/nova/db/migration.py b/nova/db/migration.py index 58f46fbf0635..0575b4528edf 100644 --- a/nova/db/migration.py +++ b/nova/db/migration.py @@ -21,19 +21,19 @@ from nova.db.sqlalchemy import migration IMPL = migration -def db_sync(version=None): +def db_sync(version=None, database='main'): """Migrate the database to `version` or the most recent version.""" - return IMPL.db_sync(version=version) + return IMPL.db_sync(version=version, database=database) -def db_version(): +def db_version(database='main'): """Display the current database version.""" - return IMPL.db_version() + return IMPL.db_version(database=database) -def db_initial_version(): +def db_initial_version(database='main'): """The starting version for the database.""" - return IMPL.db_initial_version() + return IMPL.db_initial_version(database=database) def db_null_instance_uuid_scan(delete=False): diff --git a/nova/db/sqlalchemy/api_migrations/__init__.py b/nova/db/sqlalchemy/api_migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/db/sqlalchemy/api_migrations/migrate_repo/README b/nova/db/sqlalchemy/api_migrations/migrate_repo/README new file mode 100644 index 000000000000..6218f8cac424 --- /dev/null +++ b/nova/db/sqlalchemy/api_migrations/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/nova/db/sqlalchemy/api_migrations/migrate_repo/__init__.py b/nova/db/sqlalchemy/api_migrations/migrate_repo/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/db/sqlalchemy/api_migrations/migrate_repo/migrate.cfg b/nova/db/sqlalchemy/api_migrations/migrate_repo/migrate.cfg new file mode 100644 index 000000000000..3e2ccef01684 --- /dev/null +++ b/nova/db/sqlalchemy/api_migrations/migrate_repo/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=nova_api + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] diff --git a/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/__init__.py b/nova/db/sqlalchemy/api_migrations/migrate_repo/versions/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/db/sqlalchemy/migration.py b/nova/db/sqlalchemy/migration.py index 34e192c7f2a9..f52ea5c73eeb 100644 --- a/nova/db/sqlalchemy/migration.py +++ b/nova/db/sqlalchemy/migration.py @@ -28,23 +28,25 @@ from nova.db.sqlalchemy import api as db_session from nova import exception from nova.i18n import _ -INIT_VERSION = 215 -_REPOSITORY = None +INIT_VERSION = {} +INIT_VERSION['main'] = 215 +INIT_VERSION['api'] = 0 +_REPOSITORY = {} LOG = logging.getLogger(__name__) get_engine = db_session.get_engine -def db_sync(version=None): +def db_sync(version=None, database='main'): if version is not None: try: version = int(version) except ValueError: raise exception.NovaException(_("version should be an integer")) - current_version = db_version() - repository = _find_migrate_repo() + current_version = db_version(database) + repository = _find_migrate_repo(database) if version is None or version > current_version: return versioning_api.upgrade(get_engine(), repository, version) else: @@ -52,8 +54,8 @@ def db_sync(version=None): version) -def db_version(): - repository = _find_migrate_repo() +def db_version(database='main'): + repository = _find_migrate_repo(database) try: return versioning_api.db_version(get_engine(), repository) except versioning_exceptions.DatabaseNotControlledError as exc: @@ -62,7 +64,7 @@ def db_version(): meta.reflect(bind=engine) tables = meta.tables if len(tables) == 0: - db_version_control(INIT_VERSION) + db_version_control(INIT_VERSION[database]) return versioning_api.db_version(get_engine(), repository) else: LOG.exception(exc) @@ -72,8 +74,8 @@ def db_version(): _("Upgrade DB using Essex release first.")) -def db_initial_version(): - return INIT_VERSION +def db_initial_version(database='main'): + return INIT_VERSION[database] def _process_null_records(table, col_name, check_fkeys, delete=False): @@ -151,12 +153,15 @@ def db_version_control(version=None): return version -def _find_migrate_repo(): +def _find_migrate_repo(database='main'): """Get the path for the migrate repository.""" global _REPOSITORY + rel_path = 'migrate_repo' + if database == 'api': + rel_path = os.path.join('api_migrations', 'migrate_repo') path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') + rel_path) assert os.path.exists(path) - if _REPOSITORY is None: - _REPOSITORY = Repository(path) - return _REPOSITORY + if _REPOSITORY.get(database) is None: + _REPOSITORY[database] = Repository(path) + return _REPOSITORY[database] diff --git a/nova/tests/functional/db/__init__.py b/nova/tests/functional/db/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/tests/functional/db/api/__init__.py b/nova/tests/functional/db/api/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/tests/functional/db/api/test_migrations.py b/nova/tests/functional/db/api/test_migrations.py new file mode 100644 index 000000000000..2c21b9ba240a --- /dev/null +++ b/nova/tests/functional/db/api/test_migrations.py @@ -0,0 +1,149 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Tests for database migrations. +There are "opportunistic" tests which allows testing against all 3 databases +(sqlite in memory, mysql, pg) in a properly configured unit test environment. + +For the opportunistic testing you need to set up db's named 'openstack_citest' +with user 'openstack_citest' and password 'openstack_citest' on localhost. The +test will then use that db and u/p combo to run the tests. + +For postgres on Ubuntu this can be done with the following commands:: + +| sudo -u postgres psql +| postgres=# create user openstack_citest with createdb login password +| 'openstack_citest'; +| postgres=# create database openstack_citest with owner openstack_citest; + +""" + +import logging +import os + +from migrate.versioning import repository +import mock +from oslo_config import cfg +from oslo_db.sqlalchemy import test_base +from oslo_db.sqlalchemy import test_migrations + +from nova.db import migration +from nova.db.sqlalchemy.api_migrations import migrate_repo +from nova.db.sqlalchemy import migration as sa_migration +from nova import test + + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class NovaAPIModelsSync(test_migrations.ModelsMigrationsSync): + """Test that the models match the database after migrations are run.""" + + def db_sync(self, engine): + with mock.patch.object(sa_migration, 'get_engine', + return_value=engine): + sa_migration.db_sync(database='api') + + @property + def migrate_engine(self): + return self.engine + + def get_engine(self): + return self.migrate_engine + + def test_models_sync(self): + # TODO(alaski): Remove this override to run the test when there are + # models + pass + + def get_metadata(self): + # TODO(alaski): Add model metadata once the first model is defined + pass + + def include_object(self, object_, name, type_, reflected, compare_to): + if type_ == 'table': + # migrate_version is a sqlalchemy-migrate control table and + # isn't included in the model. + if name == 'migrate_version': + return False + + return True + + +class TestNovaAPIMigrationsSQLite(NovaAPIModelsSync, + test_base.DbTestCase, + test.NoDBTestCase): + pass + + +class TestNovaAPIMigrationsMySQL(NovaAPIModelsSync, + test_base.MySQLOpportunisticTestCase, + test.NoDBTestCase): + pass + + +class TestNovaAPIMigrationsPostgreSQL(NovaAPIModelsSync, + test_base.PostgreSQLOpportunisticTestCase, test.NoDBTestCase): + pass + + +class NovaAPIMigrationsWalk(test_migrations.WalkVersionsMixin): + snake_walk = True + downgrade = True + + def setUp(self): + super(NovaAPIMigrationsWalk, self).setUp() + # NOTE(viktors): We should reduce log output because it causes issues, + # when we run tests with testr + migrate_log = logging.getLogger('migrate') + old_level = migrate_log.level + migrate_log.setLevel(logging.WARN) + self.addCleanup(migrate_log.setLevel, old_level) + + @property + def INIT_VERSION(self): + return migration.db_initial_version('api') + + @property + def REPOSITORY(self): + return repository.Repository( + os.path.abspath(os.path.dirname(migrate_repo.__file__))) + + @property + def migration_api(self): + return sa_migration.versioning_api + + @property + def migrate_engine(self): + return self.engine + + def test_walk_versions(self): + self.walk_versions(self.snake_walk, self.downgrade) + + +class TestNovaAPIMigrationsWalkSQLite(NovaAPIMigrationsWalk, + test_base.DbTestCase, + test.NoDBTestCase): + pass + + +class TestNovaAPIMigrationsWalkMySQL(NovaAPIMigrationsWalk, + test_base.MySQLOpportunisticTestCase, + test.NoDBTestCase): + pass + + +class TestNovaAPIMigrationsWalkPostgreSQL(NovaAPIMigrationsWalk, + test_base.PostgreSQLOpportunisticTestCase, test.NoDBTestCase): + pass