Add migration script to change backend to store

As glance is changing metadata value of location from 'backend' to
'store', adding migration script which will update old image locations
to use 'store' as a metadata value.

NOTE:
Bump master (train development) to 19

Change-Id: I1386c535bc8ff4519e6b0bb879026b05c930b791
Sem-Ver: api-break
This commit is contained in:
Abhishek Kekane 2019-06-17 08:06:15 +00:00
parent 733095a899
commit b4e3cb65ad
6 changed files with 236 additions and 3 deletions

View File

@ -47,9 +47,9 @@ def get_backend():
# Migration-related constants
EXPAND_BRANCH = 'expand'
CONTRACT_BRANCH = 'contract'
CURRENT_RELEASE = 'stein'
CURRENT_RELEASE = 'train'
ALEMBIC_INIT_VERSION = 'liberty'
LATEST_REVISION = 'queens_contract01'
LATEST_REVISION = 'rocky_contract02'
INIT_VERSION = 0
MIGRATE_REPO_PATH = os.path.join(

View File

@ -0,0 +1,42 @@
# Copyright 2019 RedHat Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def has_migrations(engine):
"""Returns true if at least one data row can be migrated.
There are rows left to migrate if meta_data column has
{"backend": "...."}
Note: This method can return a false positive if data migrations
are running in the background as it's being called.
"""
sql_query = ("select meta_data from image_locations where "
"INSTR(meta_data, '\"backend\":') > 0")
with engine.connect() as con:
metadata_backend = con.execute(sql_query)
if metadata_backend.rowcount > 0:
return True
return False
def migrate(engine):
"""Replace 'backend' with 'store' in meta_data column of image_locations"""
sql_query = ("UPDATE image_locations SET meta_data = REPLACE(meta_data, "
"'\"backend\":', '\"store\":') where INSTR(meta_data, "
" '\"backend\":') > 0")
with engine.connect() as con:
migrated_rows = con.execute(sql_query)
return migrated_rows.rowcount

View File

@ -0,0 +1,25 @@
# Copyright (C) 2019 RedHat Inc
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# revision identifiers, used by Alembic.
revision = 'train_contract01'
down_revision = 'rocky_contract02'
branch_labels = None
depends_on = 'train_expand01'
def upgrade():
pass

View File

@ -0,0 +1,30 @@
# Copyright (C) 2019 RedHat Inc
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""empty expand for symmetry with train_contract01
Revision ID: train_expand01
Revises: rocky_expand02
Create Date: 2019-06-17 11:55:16.657499
"""
# revision identifiers, used by Alembic.
revision = 'train_expand01'
down_revision = 'rocky_expand02'
branch_labels = None
depends_on = None
def upgrade():
pass

View File

@ -0,0 +1,131 @@
# Copyright 2019 RedHat Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import utils as db_utils
from glance.db.sqlalchemy.alembic_migrations import data_migrations
from glance.tests.functional.db import test_migrations
class TestTrainMigrate01Mixin(test_migrations.AlembicMigrationsMixin):
def _get_revisions(self, config):
return test_migrations.AlembicMigrationsMixin._get_revisions(
self, config, head='train_expand01')
def _pre_upgrade_train_expand01(self, engine):
images = db_utils.get_table(engine, 'images')
image_locations = db_utils.get_table(engine, 'image_locations')
now = datetime.datetime.now()
# inserting a public image record
image_1 = dict(deleted=False,
created_at=now,
status='active',
min_disk=0,
min_ram=0,
visibility='public',
id='image_1')
images.insert().values(image_1).execute()
image_2 = dict(deleted=False,
created_at=now,
status='active',
min_disk=0,
min_ram=0,
visibility='public',
id='image_2')
images.insert().values(image_2).execute()
# adding records to image_locations tables
temp = dict(deleted=False,
created_at=now,
image_id='image_1',
value='image_location_1',
meta_data='{"backend": "fast"}',
id=1)
image_locations.insert().values(temp).execute()
temp = dict(deleted=False,
created_at=now,
image_id='image_2',
value='image_location_2',
meta_data='{"backend": "cheap"}',
id=2)
image_locations.insert().values(temp).execute()
def _check_train_expand01(self, engine, data):
image_locations = db_utils.get_table(engine, 'image_locations')
# check that meta_data has 'backend' key for existing image_locations
rows = (image_locations.select()
.order_by(image_locations.c.id)
.execute()
.fetchall())
self.assertEqual(2, len(rows))
for row in rows:
self.assertIn('"backend":', row['meta_data'])
# run data migrations
data_migrations.migrate(engine)
# check that meta_data has 'backend' key replaced with 'store'
rows = (image_locations.select()
.order_by(image_locations.c.id)
.execute()
.fetchall())
self.assertEqual(2, len(rows))
for row in rows:
self.assertNotIn('"backend":', row['meta_data'])
self.assertIn('"store":', row['meta_data'])
class TestTrainMigrate01MySQL(TestTrainMigrate01Mixin,
test_base.MySQLOpportunisticTestCase):
pass
class TestTrainMigrate01_EmptyDBMixin(test_migrations.AlembicMigrationsMixin):
"""This mixin is used to create an initial glance database and upgrade it
up to the train_expand01 revision.
"""
def _get_revisions(self, config):
return test_migrations.AlembicMigrationsMixin._get_revisions(
self, config, head='train_expand01')
def _pre_upgrade_train_expand01(self, engine):
# New/empty database
pass
def _check_train_expand01(self, engine, data):
images = db_utils.get_table(engine, 'images')
# check that there are no rows in the images table
rows = (images.select()
.order_by(images.c.id)
.execute()
.fetchall())
self.assertEqual(0, len(rows))
# run data migrations
data_migrations.migrate(engine)
class TestTrainMigrate01_EmptyDBMySQL(TestTrainMigrate01_EmptyDBMixin,
test_base.MySQLOpportunisticTestCase):
"""This test runs the Train data migrations on an empty databse."""
pass

View File

@ -378,6 +378,9 @@ class TestManage(TestManageBase):
' revision. But, current revisions are: test ',
exit.code)
@mock.patch(
'glance.db.sqlalchemy.alembic_migrations.data_migrations.'
'has_pending_migrations')
@mock.patch(
'glance.db.sqlalchemy.alembic_migrations.get_current_alembic_heads')
@mock.patch(
@ -386,13 +389,15 @@ class TestManage(TestManageBase):
@mock.patch.object(manage.DbCommands, '_sync')
def test_contract(self, mock_sync, mock_validate_engine,
mock_get_alembic_branch_head,
mock_get_current_alembic_heads):
mock_get_current_alembic_heads,
mock_has_pending_migrations):
engine = mock_validate_engine.return_value
engine.engine.name = 'mysql'
mock_get_current_alembic_heads.side_effect = ['pike_expand01',
'pike_contract01']
mock_get_alembic_branch_head.side_effect = ['pike_contract01',
'pike_expand01']
mock_has_pending_migrations.return_value = False
self.db.contract()
mock_sync.assert_called_once_with(version='pike_contract01')