diff --git a/glance/db/migration.py b/glance/db/migration.py index 79d458427a..b85866b8e0 100644 --- a/glance/db/migration.py +++ b/glance/db/migration.py @@ -47,9 +47,9 @@ def get_backend(): # Migration-related constants EXPAND_BRANCH = 'expand' CONTRACT_BRANCH = 'contract' -CURRENT_RELEASE = 'stein' +CURRENT_RELEASE = 'train' ALEMBIC_INIT_VERSION = 'liberty' -LATEST_REVISION = 'queens_contract01' +LATEST_REVISION = 'rocky_contract02' INIT_VERSION = 0 MIGRATE_REPO_PATH = os.path.join( diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/train_migrate01_backend_to_store.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/train_migrate01_backend_to_store.py new file mode 100644 index 0000000000..cfb2b82088 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/data_migrations/train_migrate01_backend_to_store.py @@ -0,0 +1,42 @@ +# Copyright 2019 RedHat Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def has_migrations(engine): + """Returns true if at least one data row can be migrated. + + There are rows left to migrate if meta_data column has + {"backend": "...."} + + Note: This method can return a false positive if data migrations + are running in the background as it's being called. + """ + sql_query = ("select meta_data from image_locations where " + "INSTR(meta_data, '\"backend\":') > 0") + with engine.connect() as con: + metadata_backend = con.execute(sql_query) + if metadata_backend.rowcount > 0: + return True + + return False + + +def migrate(engine): + """Replace 'backend' with 'store' in meta_data column of image_locations""" + sql_query = ("UPDATE image_locations SET meta_data = REPLACE(meta_data, " + "'\"backend\":', '\"store\":') where INSTR(meta_data, " + " '\"backend\":') > 0") + with engine.connect() as con: + migrated_rows = con.execute(sql_query) + return migrated_rows.rowcount diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/train_contract01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/train_contract01_empty.py new file mode 100644 index 0000000000..eaa6b546da --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/train_contract01_empty.py @@ -0,0 +1,25 @@ +# Copyright (C) 2019 RedHat Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# revision identifiers, used by Alembic. +revision = 'train_contract01' +down_revision = 'rocky_contract02' +branch_labels = None +depends_on = 'train_expand01' + + +def upgrade(): + pass diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/train_expand01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/train_expand01_empty.py new file mode 100644 index 0000000000..d8fd277d8b --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/train_expand01_empty.py @@ -0,0 +1,30 @@ +# Copyright (C) 2019 RedHat Inc +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""empty expand for symmetry with train_contract01 + +Revision ID: train_expand01 +Revises: rocky_expand02 +Create Date: 2019-06-17 11:55:16.657499 + +""" + +# revision identifiers, used by Alembic. +revision = 'train_expand01' +down_revision = 'rocky_expand02' +branch_labels = None +depends_on = None + + +def upgrade(): + pass diff --git a/glance/tests/functional/db/migrations/test_train_migrate01.py b/glance/tests/functional/db/migrations/test_train_migrate01.py new file mode 100644 index 0000000000..6d13eb753c --- /dev/null +++ b/glance/tests/functional/db/migrations/test_train_migrate01.py @@ -0,0 +1,131 @@ +# Copyright 2019 RedHat Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +from oslo_db.sqlalchemy import test_base +from oslo_db.sqlalchemy import utils as db_utils + +from glance.db.sqlalchemy.alembic_migrations import data_migrations +from glance.tests.functional.db import test_migrations + + +class TestTrainMigrate01Mixin(test_migrations.AlembicMigrationsMixin): + + def _get_revisions(self, config): + return test_migrations.AlembicMigrationsMixin._get_revisions( + self, config, head='train_expand01') + + def _pre_upgrade_train_expand01(self, engine): + images = db_utils.get_table(engine, 'images') + image_locations = db_utils.get_table(engine, 'image_locations') + now = datetime.datetime.now() + + # inserting a public image record + image_1 = dict(deleted=False, + created_at=now, + status='active', + min_disk=0, + min_ram=0, + visibility='public', + id='image_1') + images.insert().values(image_1).execute() + + image_2 = dict(deleted=False, + created_at=now, + status='active', + min_disk=0, + min_ram=0, + visibility='public', + id='image_2') + images.insert().values(image_2).execute() + + # adding records to image_locations tables + temp = dict(deleted=False, + created_at=now, + image_id='image_1', + value='image_location_1', + meta_data='{"backend": "fast"}', + id=1) + image_locations.insert().values(temp).execute() + + temp = dict(deleted=False, + created_at=now, + image_id='image_2', + value='image_location_2', + meta_data='{"backend": "cheap"}', + id=2) + image_locations.insert().values(temp).execute() + + def _check_train_expand01(self, engine, data): + image_locations = db_utils.get_table(engine, 'image_locations') + + # check that meta_data has 'backend' key for existing image_locations + rows = (image_locations.select() + .order_by(image_locations.c.id) + .execute() + .fetchall()) + self.assertEqual(2, len(rows)) + for row in rows: + self.assertIn('"backend":', row['meta_data']) + + # run data migrations + data_migrations.migrate(engine) + + # check that meta_data has 'backend' key replaced with 'store' + rows = (image_locations.select() + .order_by(image_locations.c.id) + .execute() + .fetchall()) + self.assertEqual(2, len(rows)) + for row in rows: + self.assertNotIn('"backend":', row['meta_data']) + self.assertIn('"store":', row['meta_data']) + + +class TestTrainMigrate01MySQL(TestTrainMigrate01Mixin, + test_base.MySQLOpportunisticTestCase): + pass + + +class TestTrainMigrate01_EmptyDBMixin(test_migrations.AlembicMigrationsMixin): + """This mixin is used to create an initial glance database and upgrade it + up to the train_expand01 revision. + """ + def _get_revisions(self, config): + return test_migrations.AlembicMigrationsMixin._get_revisions( + self, config, head='train_expand01') + + def _pre_upgrade_train_expand01(self, engine): + # New/empty database + pass + + def _check_train_expand01(self, engine, data): + images = db_utils.get_table(engine, 'images') + + # check that there are no rows in the images table + rows = (images.select() + .order_by(images.c.id) + .execute() + .fetchall()) + self.assertEqual(0, len(rows)) + + # run data migrations + data_migrations.migrate(engine) + + +class TestTrainMigrate01_EmptyDBMySQL(TestTrainMigrate01_EmptyDBMixin, + test_base.MySQLOpportunisticTestCase): + """This test runs the Train data migrations on an empty databse.""" + pass diff --git a/glance/tests/unit/test_manage.py b/glance/tests/unit/test_manage.py index 4da916bdad..f193d9171b 100644 --- a/glance/tests/unit/test_manage.py +++ b/glance/tests/unit/test_manage.py @@ -378,6 +378,9 @@ class TestManage(TestManageBase): ' revision. But, current revisions are: test ', exit.code) + @mock.patch( + 'glance.db.sqlalchemy.alembic_migrations.data_migrations.' + 'has_pending_migrations') @mock.patch( 'glance.db.sqlalchemy.alembic_migrations.get_current_alembic_heads') @mock.patch( @@ -386,13 +389,15 @@ class TestManage(TestManageBase): @mock.patch.object(manage.DbCommands, '_sync') def test_contract(self, mock_sync, mock_validate_engine, mock_get_alembic_branch_head, - mock_get_current_alembic_heads): + mock_get_current_alembic_heads, + mock_has_pending_migrations): engine = mock_validate_engine.return_value engine.engine.name = 'mysql' mock_get_current_alembic_heads.side_effect = ['pike_expand01', 'pike_contract01'] mock_get_alembic_branch_head.side_effect = ['pike_contract01', 'pike_expand01'] + mock_has_pending_migrations.return_value = False self.db.contract() mock_sync.assert_called_once_with(version='pike_contract01')