Browse Source

Add expand/migrate/contract migrations for CI

This patch adds equivalent expand/migrate/contract migration scripts for
Community Images. The expand migration
'ocata_expand01_add_visibility.py' creates a new migration branch 'expand'
from the last known migration i.e., 'mitaka02'. Similarly, the contract
migration 'ocata_contract01_drop_is_public.py' creates another new
migration branch called 'contract' from the last known migration.

The data migration 'ocata_migrate01_community_images.py' migrates
all rows in the database at once. There is possibility of performance
degradation while the data migrations are running.

Change-Id: I34f5623d6804e9fe594e6b5b196ea4a162578196
Partially-Implements: blueprint database-strategy-for-rolling-upgrades
Co-Authored-By: Hemanth Makkapati <hemanth.makkapati@rackspace.com>
Depends-On: Ie839e0f240436dce7b151de5b464373516ff5a64
tags/14.0.0.0rc1
Hemanth Makkapati 2 years ago
parent
commit
9859df2d1f

+ 15
- 0
glance/cmd/manage.py View File

@@ -129,6 +129,11 @@ class DbCommands(object):
129 129
 
130 130
     def expand(self):
131 131
         """Run the expansion phase of a rolling upgrade procedure."""
132
+        engine = db_api.get_engine()
133
+        if engine.engine.name != 'mysql':
134
+            sys.exit(_('Rolling upgrades are currently supported only for '
135
+                       'MySQL'))
136
+
132 137
         expand_head = alembic_migrations.get_alembic_branch_head(
133 138
             db_migration.EXPAND_BRANCH)
134 139
         if not expand_head:
@@ -146,6 +151,11 @@ class DbCommands(object):
146 151
 
147 152
     def contract(self):
148 153
         """Run the contraction phase of a rolling upgrade procedure."""
154
+        engine = db_api.get_engine()
155
+        if engine.engine.name != 'mysql':
156
+            sys.exit(_('Rolling upgrades are currently supported only for '
157
+                       'MySQL'))
158
+
149 159
         contract_head = alembic_migrations.get_alembic_branch_head(
150 160
             db_migration.CONTRACT_BRANCH)
151 161
         if not contract_head:
@@ -178,6 +188,11 @@ class DbCommands(object):
178 188
                                             'curr_revs': curr_heads})
179 189
 
180 190
     def migrate(self):
191
+        engine = db_api.get_engine()
192
+        if engine.engine.name != 'mysql':
193
+            sys.exit(_('Rolling upgrades are currently supported only for '
194
+                       'MySQL'))
195
+
181 196
         curr_heads = alembic_migrations.get_current_alembic_heads()
182 197
         expand_head = alembic_migrations.get_alembic_branch_head(
183 198
             db_migration.EXPAND_BRANCH)

+ 103
- 0
glance/db/sqlalchemy/alembic_migrations/data_migrations/ocata_migrate01_community_images.py View File

@@ -0,0 +1,103 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2016 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+from sqlalchemy import MetaData, select, Table, and_, not_
17
+
18
+
19
+def has_migrations(engine):
20
+    """Returns true if at least one data row can be migrated.
21
+
22
+    There are rows left to migrate if:
23
+     #1 There exists a row with visibility not set yet.
24
+        Or
25
+     #2 There exists a private image with active members but its visibility
26
+        isn't set to 'shared' yet.
27
+
28
+    Note: This method can return a false positive if data migrations
29
+    are running in the background as it's being called.
30
+    """
31
+    meta = MetaData(engine)
32
+    images = Table('images', meta, autoload=True)
33
+
34
+    rows_with_null_visibility = (select([images.c.id])
35
+                                 .where(images.c.visibility == None)
36
+                                 .limit(1)
37
+                                 .execute())
38
+
39
+    if rows_with_null_visibility.rowcount == 1:
40
+        return True
41
+
42
+    image_members = Table('image_members', meta, autoload=True)
43
+    rows_with_pending_shared = (select[images.c.id]
44
+                                .where(and_(
45
+                                    images.c.visibility == 'private',
46
+                                    images.c.id.in_(
47
+                                        select([image_members.c.image_id])
48
+                                        .distinct()
49
+                                        .where(not_(image_members.c.deleted))))
50
+                                       )
51
+                                .limit(1)
52
+                                .execute())
53
+    if rows_with_pending_shared.rowcount == 1:
54
+        return True
55
+
56
+    return False
57
+
58
+
59
+def _mark_all_public_images_with_public_visibility(images):
60
+    migrated_rows = (images
61
+                     .update().values(visibility='public')
62
+                     .where(images.c.is_public)
63
+                     .execute())
64
+    return migrated_rows.rowcount
65
+
66
+
67
+def _mark_all_non_public_images_with_private_visibility(images):
68
+    migrated_rows = (images
69
+                     .update().values(visibility='private')
70
+                     .where(not_(images.c.is_public))
71
+                     .execute())
72
+    return migrated_rows.rowcount
73
+
74
+
75
+def _mark_all_private_images_with_members_as_shared_visibility(images,
76
+                                                               image_members):
77
+    migrated_rows = (images
78
+                     .update().values(visibility='shared')
79
+                     .where(and_(images.c.visibility == 'private',
80
+                                 images.c.id.in_(
81
+                                     select([image_members.c.image_id])
82
+                                     .distinct()
83
+                                     .where(not_(image_members.c.deleted)))))
84
+                     .execute())
85
+    return migrated_rows.rowcount
86
+
87
+
88
+def _migrate_all(engine):
89
+    meta = MetaData(engine)
90
+    images = Table('images', meta, autoload=True)
91
+    image_members = Table('image_members', meta, autoload=True)
92
+
93
+    num_rows = _mark_all_public_images_with_public_visibility(images)
94
+    num_rows += _mark_all_non_public_images_with_private_visibility(images)
95
+    num_rows += _mark_all_private_images_with_members_as_shared_visibility(
96
+        images, image_members)
97
+
98
+    return num_rows
99
+
100
+
101
+def migrate(engine):
102
+    """Set visibility column based on is_public and image members."""
103
+    return _migrate_all(engine)

+ 67
- 0
glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py View File

@@ -0,0 +1,67 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+"""remove is_public from images
14
+
15
+Revision ID: ocata_contract01
16
+Revises: mitaka02
17
+Create Date: 2017-01-27 12:58:16.647499
18
+
19
+"""
20
+
21
+from alembic import op
22
+from sqlalchemy import MetaData, Table
23
+
24
+from glance.db import migration
25
+
26
+# revision identifiers, used by Alembic.
27
+revision = 'ocata_contract01'
28
+down_revision = 'mitaka02'
29
+branch_labels = migration.CONTRACT_BRANCH
30
+depends_on = 'expand'
31
+
32
+
33
+MYSQL_DROP_INSERT_TRIGGER = """
34
+DROP TRIGGER insert_visibility;
35
+"""
36
+
37
+MYSQL_DROP_UPDATE_TRIGGER = """
38
+DROP TRIGGER update_visibility;
39
+"""
40
+
41
+
42
+def _drop_column():
43
+    op.drop_index('ix_images_is_public', 'images')
44
+    op.drop_column('images', 'is_public')
45
+
46
+
47
+def _drop_triggers(engine):
48
+    engine_name = engine.engine.name
49
+    if engine_name == "mysql":
50
+        op.execute(MYSQL_DROP_INSERT_TRIGGER)
51
+        op.execute(MYSQL_DROP_UPDATE_TRIGGER)
52
+
53
+
54
+def _set_nullability_and_default_on_visibility(meta):
55
+    # NOTE(hemanthm): setting the default on 'visibility' column
56
+    # to 'shared'. Also, marking it as non-nullable.
57
+    images = Table('images', meta, autoload=True)
58
+    images.c.visibility.alter(nullable=False, server_default='shared')
59
+
60
+
61
+def upgrade():
62
+    migrate_engine = op.get_bind()
63
+    meta = MetaData(bind=migrate_engine)
64
+
65
+    _drop_column()
66
+    _drop_triggers(migrate_engine)
67
+    _set_nullability_and_default_on_visibility(meta)

+ 151
- 0
glance/db/sqlalchemy/alembic_migrations/versions/ocata_expand01_add_visibility.py View File

@@ -0,0 +1,151 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+"""add visibility to images
14
+
15
+Revision ID: ocata_expand01
16
+Revises: mitaka02
17
+Create Date: 2017-01-27 12:58:16.647499
18
+
19
+"""
20
+
21
+from alembic import op
22
+from sqlalchemy import Column, Enum, MetaData, Table
23
+
24
+from glance.db import migration
25
+
26
+# revision identifiers, used by Alembic.
27
+revision = 'ocata_expand01'
28
+down_revision = 'mitaka02'
29
+branch_labels = migration.EXPAND_BRANCH
30
+depends_on = None
31
+
32
+ERROR_MESSAGE = 'Invalid visibility value'
33
+MYSQL_INSERT_TRIGGER = """
34
+CREATE TRIGGER insert_visibility BEFORE INSERT ON images
35
+FOR EACH ROW
36
+BEGIN
37
+    -- NOTE(abashmak):
38
+    -- The following IF/ELSE block implements a priority decision tree.
39
+    -- Strict order MUST be followed to correctly cover all the edge cases.
40
+
41
+    -- Edge case: neither is_public nor visibility specified
42
+    --            (or both specified as NULL):
43
+    IF NEW.is_public <=> NULL AND NEW.visibility <=> NULL THEN
44
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
45
+    -- Edge case: both is_public and visibility specified:
46
+    ELSEIF NOT(NEW.is_public <=> NULL OR NEW.visibility <=> NULL) THEN
47
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
48
+    -- Inserting with is_public, set visibility accordingly:
49
+    ELSEIF NOT NEW.is_public <=> NULL THEN
50
+        IF NEW.is_public = 1 THEN
51
+            SET NEW.visibility = 'public';
52
+        ELSE
53
+            SET NEW.visibility = 'shared';
54
+        END IF;
55
+    -- Inserting with visibility, set is_public accordingly:
56
+    ELSEIF NOT NEW.visibility <=> NULL THEN
57
+        IF NEW.visibility = 'public' THEN
58
+            SET NEW.is_public = 1;
59
+        ELSE
60
+            SET NEW.is_public = 0;
61
+        END IF;
62
+    -- Edge case: either one of: is_public or visibility,
63
+    --            is explicitly set to NULL:
64
+    ELSE
65
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
66
+    END IF;
67
+END;
68
+"""
69
+
70
+MYSQL_UPDATE_TRIGGER = """
71
+CREATE TRIGGER update_visibility BEFORE UPDATE ON images
72
+FOR EACH ROW
73
+BEGIN
74
+    -- Case: new value specified for is_public:
75
+    IF NOT NEW.is_public <=> OLD.is_public THEN
76
+        -- Edge case: is_public explicitly set to NULL:
77
+        IF NEW.is_public <=> NULL THEN
78
+            SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
79
+        -- Edge case: new value also specified for visibility
80
+        ELSEIF NOT NEW.visibility <=> OLD.visibility THEN
81
+            SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
82
+        -- Case: visibility not specified or specified as OLD value:
83
+        -- NOTE(abashmak): There is no way to reliably determine which
84
+        -- of the above two cases occurred, but allowing to proceed with
85
+        -- the update in either case does not break the model for both
86
+        -- N and N-1 services.
87
+        ELSE
88
+            -- Set visibility according to the value of is_public:
89
+            IF NEW.is_public <=> 1 THEN
90
+                SET NEW.visibility = 'public';
91
+            ELSE
92
+                SET NEW.visibility = 'shared';
93
+            END IF;
94
+        END IF;
95
+    -- Case: new value specified for visibility:
96
+    ELSEIF NOT NEW.visibility <=> OLD.visibility THEN
97
+        -- Edge case: visibility explicitly set to NULL:
98
+        IF NEW.visibility <=> NULL THEN
99
+            SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
100
+        -- Edge case: new value also specified for is_public
101
+        ELSEIF NOT NEW.is_public <=> OLD.is_public THEN
102
+            SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
103
+        -- Case: is_public not specified or specified as OLD value:
104
+        -- NOTE(abashmak): There is no way to reliably determine which
105
+        -- of the above two cases occurred, but allowing to proceed with
106
+        -- the update in either case does not break the model for both
107
+        -- N and N-1 services.
108
+        ELSE
109
+            -- Set is_public according to the value of visibility:
110
+            IF NEW.visibility <=> 'public' THEN
111
+                SET NEW.is_public = 1;
112
+            ELSE
113
+                SET NEW.is_public = 0;
114
+            END IF;
115
+        END IF;
116
+    END IF;
117
+END;
118
+"""
119
+
120
+
121
+def _add_visibility_column(meta):
122
+    enum = Enum('private', 'public', 'shared', 'community', metadata=meta,
123
+                name='image_visibility')
124
+    enum.create()
125
+    v_col = Column('visibility', enum, nullable=True, server_default=None)
126
+    op.add_column('images', v_col)
127
+    op.create_index('visibility_image_idx', 'images', ['visibility'])
128
+
129
+
130
+def _add_triggers(engine):
131
+    if engine.engine.name == 'mysql':
132
+        op.execute(MYSQL_INSERT_TRIGGER % (ERROR_MESSAGE, ERROR_MESSAGE,
133
+                                           ERROR_MESSAGE))
134
+        op.execute(MYSQL_UPDATE_TRIGGER % (ERROR_MESSAGE, ERROR_MESSAGE,
135
+                                           ERROR_MESSAGE, ERROR_MESSAGE))
136
+
137
+
138
+def _change_nullability_and_default_on_is_public(meta):
139
+    # NOTE(hemanthm): we mark is_public as nullable so that when new versions
140
+    # add data only to be visibility column, is_public can be null.
141
+    images = Table('images', meta, autoload=True)
142
+    images.c.is_public.alter(nullable=True, server_default=None)
143
+
144
+
145
+def upgrade():
146
+    migrate_engine = op.get_bind()
147
+    meta = MetaData(bind=migrate_engine)
148
+
149
+    _add_visibility_column(meta)
150
+    _change_nullability_and_default_on_is_public(meta)
151
+    _add_triggers(migrate_engine)

+ 64
- 0
glance/tests/functional/db/migrations/test_ocata_contract01.py View File

@@ -0,0 +1,64 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+import datetime
14
+
15
+from oslo_db.sqlalchemy import test_base
16
+from oslo_db.sqlalchemy import utils as db_utils
17
+
18
+from glance.tests.functional.db import test_migrations
19
+
20
+
21
+class TestOcataContract01Mixin(test_migrations.AlembicMigrationsMixin):
22
+
23
+    def _get_revisions(self, config):
24
+        return test_migrations.AlembicMigrationsMixin._get_revisions(
25
+            self, config, head='ocata_contract01')
26
+
27
+    def _pre_upgrade_ocata_contract01(self, engine):
28
+        images = db_utils.get_table(engine, 'images')
29
+        now = datetime.datetime.now()
30
+        self.assertIn('is_public', images.c)
31
+        self.assertIn('visibility', images.c)
32
+        self.assertTrue(images.c.is_public.nullable)
33
+        self.assertTrue(images.c.visibility.nullable)
34
+
35
+        # inserting a public image record
36
+        public_temp = dict(deleted=False,
37
+                           created_at=now,
38
+                           status='active',
39
+                           is_public=True,
40
+                           min_disk=0,
41
+                           min_ram=0,
42
+                           id='public_id_before_expand')
43
+        images.insert().values(public_temp).execute()
44
+
45
+        # inserting a private image record
46
+        shared_temp = dict(deleted=False,
47
+                           created_at=now,
48
+                           status='active',
49
+                           is_public=False,
50
+                           min_disk=0,
51
+                           min_ram=0,
52
+                           id='private_id_before_expand')
53
+        images.insert().values(shared_temp).execute()
54
+
55
+    def _check_ocata_contract01(self, engine, data):
56
+        # check that after contract 'is_public' column is dropped
57
+        images = db_utils.get_table(engine, 'images')
58
+        self.assertNotIn('is_public', images.c)
59
+        self.assertIn('visibility', images.c)
60
+
61
+
62
+class TestOcataContract01MySQL(TestOcataContract01Mixin,
63
+                               test_base.MySQLOpportunisticTestCase):
64
+    pass

+ 174
- 0
glance/tests/functional/db/migrations/test_ocata_expand01.py View File

@@ -0,0 +1,174 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+import datetime
14
+
15
+from oslo_db.sqlalchemy import test_base
16
+from oslo_db.sqlalchemy import utils as db_utils
17
+
18
+from glance.tests.functional.db import test_migrations
19
+
20
+
21
+class TestOcataExpand01Mixin(test_migrations.AlembicMigrationsMixin):
22
+
23
+    def _get_revisions(self, config):
24
+        return test_migrations.AlembicMigrationsMixin._get_revisions(
25
+            self, config, head='ocata_expand01')
26
+
27
+    def _pre_upgrade_ocata_expand01(self, engine):
28
+        images = db_utils.get_table(engine, 'images')
29
+        now = datetime.datetime.now()
30
+        self.assertIn('is_public', images.c)
31
+        self.assertNotIn('visibility', images.c)
32
+        self.assertFalse(images.c.is_public.nullable)
33
+
34
+        # inserting a public image record
35
+        public_temp = dict(deleted=False,
36
+                           created_at=now,
37
+                           status='active',
38
+                           is_public=True,
39
+                           min_disk=0,
40
+                           min_ram=0,
41
+                           id='public_id_before_expand')
42
+        images.insert().values(public_temp).execute()
43
+
44
+        # inserting a private image record
45
+        shared_temp = dict(deleted=False,
46
+                           created_at=now,
47
+                           status='active',
48
+                           is_public=False,
49
+                           min_disk=0,
50
+                           min_ram=0,
51
+                           id='private_id_before_expand')
52
+        images.insert().values(shared_temp).execute()
53
+
54
+    def _check_ocata_expand01(self, engine, data):
55
+        # check that after migration, 'visibility' column is introduced
56
+        images = db_utils.get_table(engine, 'images')
57
+        self.assertIn('visibility', images.c)
58
+        self.assertIn('is_public', images.c)
59
+        self.assertTrue(images.c.is_public.nullable)
60
+        self.assertTrue(images.c.visibility.nullable)
61
+
62
+        # tests visibility set to None for existing images
63
+        rows = (images.select()
64
+                .where(images.c.id.like('%_before_expand'))
65
+                .order_by(images.c.id)
66
+                .execute()
67
+                .fetchall())
68
+
69
+        self.assertEqual(2, len(rows))
70
+        # private image first
71
+        self.assertEqual(0, rows[0]['is_public'])
72
+        self.assertEqual('private_id_before_expand', rows[0]['id'])
73
+        self.assertIsNone(rows[0]['visibility'])
74
+        # then public image
75
+        self.assertEqual(1, rows[1]['is_public'])
76
+        self.assertEqual('public_id_before_expand', rows[1]['id'])
77
+        self.assertIsNone(rows[1]['visibility'])
78
+
79
+        self._test_trigger_old_to_new(images)
80
+        self._test_trigger_new_to_old(images)
81
+
82
+    def _test_trigger_new_to_old(self, images):
83
+        now = datetime.datetime.now()
84
+        # inserting a public image record after expand
85
+        public_temp = dict(deleted=False,
86
+                           created_at=now,
87
+                           status='active',
88
+                           visibility='public',
89
+                           min_disk=0,
90
+                           min_ram=0,
91
+                           id='public_id_new_to_old')
92
+        images.insert().values(public_temp).execute()
93
+
94
+        # inserting a private image record after expand
95
+        shared_temp = dict(deleted=False,
96
+                           created_at=now,
97
+                           status='active',
98
+                           visibility='private',
99
+                           min_disk=0,
100
+                           min_ram=0,
101
+                           id='private_id_new_to_old')
102
+        images.insert().values(shared_temp).execute()
103
+
104
+        # inserting a shared image record after expand
105
+        shared_temp = dict(deleted=False,
106
+                           created_at=now,
107
+                           status='active',
108
+                           visibility='shared',
109
+                           min_disk=0,
110
+                           min_ram=0,
111
+                           id='shared_id_new_to_old')
112
+        images.insert().values(shared_temp).execute()
113
+
114
+        # test visibility is set appropriately by the trigger for new images
115
+        rows = (images.select()
116
+                .where(images.c.id.like('%_new_to_old'))
117
+                .order_by(images.c.id)
118
+                .execute()
119
+                .fetchall())
120
+
121
+        self.assertEqual(3, len(rows))
122
+        # private image first
123
+        self.assertEqual(0, rows[0]['is_public'])
124
+        self.assertEqual('private_id_new_to_old', rows[0]['id'])
125
+        self.assertEqual('private', rows[0]['visibility'])
126
+        # then public image
127
+        self.assertEqual(1, rows[1]['is_public'])
128
+        self.assertEqual('public_id_new_to_old', rows[1]['id'])
129
+        self.assertEqual('public', rows[1]['visibility'])
130
+        # then shared image
131
+        self.assertEqual(0, rows[2]['is_public'])
132
+        self.assertEqual('shared_id_new_to_old', rows[2]['id'])
133
+        self.assertEqual('shared', rows[2]['visibility'])
134
+
135
+    def _test_trigger_old_to_new(self, images):
136
+        now = datetime.datetime.now()
137
+        # inserting a public image record after expand
138
+        public_temp = dict(deleted=False,
139
+                           created_at=now,
140
+                           status='active',
141
+                           is_public=True,
142
+                           min_disk=0,
143
+                           min_ram=0,
144
+                           id='public_id_old_to_new')
145
+        images.insert().values(public_temp).execute()
146
+        # inserting a private image record after expand
147
+        shared_temp = dict(deleted=False,
148
+                           created_at=now,
149
+                           status='active',
150
+                           is_public=False,
151
+                           min_disk=0,
152
+                           min_ram=0,
153
+                           id='private_id_old_to_new')
154
+        images.insert().values(shared_temp).execute()
155
+        # tests visibility is set appropriately by the trigger for new images
156
+        rows = (images.select()
157
+                .where(images.c.id.like('%_old_to_new'))
158
+                .order_by(images.c.id)
159
+                .execute()
160
+                .fetchall())
161
+        self.assertEqual(2, len(rows))
162
+        # private image first
163
+        self.assertEqual(0, rows[0]['is_public'])
164
+        self.assertEqual('private_id_old_to_new', rows[0]['id'])
165
+        self.assertEqual('shared', rows[0]['visibility'])
166
+        # then public image
167
+        self.assertEqual(1, rows[1]['is_public'])
168
+        self.assertEqual('public_id_old_to_new', rows[1]['id'])
169
+        self.assertEqual('public', rows[1]['visibility'])
170
+
171
+
172
+class TestOcataExpand01MySQL(TestOcataExpand01Mixin,
173
+                             test_base.MySQLOpportunisticTestCase):
174
+    pass

+ 147
- 0
glance/tests/functional/db/migrations/test_ocata_migrate01.py View File

@@ -0,0 +1,147 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+import datetime
14
+
15
+from oslo_db.sqlalchemy import test_base
16
+from oslo_db.sqlalchemy import utils as db_utils
17
+
18
+from glance.db.sqlalchemy.alembic_migrations import data_migrations
19
+from glance.tests.functional.db import test_migrations
20
+
21
+
22
+class TestOcataMigrate01Mixin(test_migrations.AlembicMigrationsMixin):
23
+
24
+    def _get_revisions(self, config):
25
+        return test_migrations.AlembicMigrationsMixin._get_revisions(
26
+            self, config, head='ocata_expand01')
27
+
28
+    def _pre_upgrade_ocata_expand01(self, engine):
29
+        images = db_utils.get_table(engine, 'images')
30
+        image_members = db_utils.get_table(engine, 'image_members')
31
+        now = datetime.datetime.now()
32
+
33
+        # inserting a public image record
34
+        public_temp = dict(deleted=False,
35
+                           created_at=now,
36
+                           status='active',
37
+                           is_public=True,
38
+                           min_disk=0,
39
+                           min_ram=0,
40
+                           id='public_id')
41
+        images.insert().values(public_temp).execute()
42
+
43
+        # inserting a non-public image record for 'shared' visibility test
44
+        shared_temp = dict(deleted=False,
45
+                           created_at=now,
46
+                           status='active',
47
+                           is_public=False,
48
+                           min_disk=0,
49
+                           min_ram=0,
50
+                           id='shared_id')
51
+        images.insert().values(shared_temp).execute()
52
+
53
+        # inserting a non-public image records for 'private' visibility test
54
+        private_temp = dict(deleted=False,
55
+                            created_at=now,
56
+                            status='active',
57
+                            is_public=False,
58
+                            min_disk=0,
59
+                            min_ram=0,
60
+                            id='private_id_1')
61
+        images.insert().values(private_temp).execute()
62
+
63
+        private_temp = dict(deleted=False,
64
+                            created_at=now,
65
+                            status='active',
66
+                            is_public=False,
67
+                            min_disk=0,
68
+                            min_ram=0,
69
+                            id='private_id_2')
70
+        images.insert().values(private_temp).execute()
71
+
72
+        # adding an active as well as a deleted image member for checking
73
+        # 'shared' visibility
74
+        temp = dict(deleted=False,
75
+                    created_at=now,
76
+                    image_id='shared_id',
77
+                    member='fake_member_452',
78
+                    can_share=True,
79
+                    id=45)
80
+        image_members.insert().values(temp).execute()
81
+
82
+        temp = dict(deleted=True,
83
+                    created_at=now,
84
+                    image_id='shared_id',
85
+                    member='fake_member_453',
86
+                    can_share=True,
87
+                    id=453)
88
+        image_members.insert().values(temp).execute()
89
+
90
+        # adding an image member, but marking it deleted,
91
+        # for testing 'private' visibility
92
+        temp = dict(deleted=True,
93
+                    created_at=now,
94
+                    image_id='private_id_2',
95
+                    member='fake_member_451',
96
+                    can_share=True,
97
+                    id=451)
98
+        image_members.insert().values(temp).execute()
99
+
100
+        # adding an active image member for the 'public' image,
101
+        # to test it remains public regardless.
102
+        temp = dict(deleted=False,
103
+                    created_at=now,
104
+                    image_id='public_id',
105
+                    member='fake_member_450',
106
+                    can_share=True,
107
+                    id=450)
108
+        image_members.insert().values(temp).execute()
109
+
110
+    def _check_ocata_expand01(self, engine, data):
111
+        images = db_utils.get_table(engine, 'images')
112
+
113
+        # check that visibility is null for existing images
114
+        rows = (images.select()
115
+                .order_by(images.c.id)
116
+                .execute()
117
+                .fetchall())
118
+        self.assertEqual(4, len(rows))
119
+        for row in rows:
120
+            self.assertIsNone(row['visibility'])
121
+
122
+        # run data migrations
123
+        data_migrations.migrate(engine)
124
+
125
+        # check that visibility is set appropriately for all images
126
+        rows = (images.select()
127
+                .order_by(images.c.id)
128
+                .execute()
129
+                .fetchall())
130
+        self.assertEqual(4, len(rows))
131
+        # private_id_1 has private visibility
132
+        self.assertEqual('private_id_1', rows[0]['id'])
133
+        self.assertEqual('private', rows[0]['visibility'])
134
+        # private_id_2 has private visibility
135
+        self.assertEqual('private_id_2', rows[1]['id'])
136
+        self.assertEqual('private', rows[1]['visibility'])
137
+        # public_id has public visibility
138
+        self.assertEqual('public_id', rows[2]['id'])
139
+        self.assertEqual('public', rows[2]['visibility'])
140
+        # shared_id has shared visibility
141
+        self.assertEqual('shared_id', rows[3]['id'])
142
+        self.assertEqual('shared', rows[3]['visibility'])
143
+
144
+
145
+class TestOcataMigrate01MySQL(TestOcataMigrate01Mixin,
146
+                              test_base.MySQLOpportunisticTestCase):
147
+    pass

+ 4
- 3
glance/tests/functional/db/test_migrations.py View File

@@ -23,7 +23,7 @@ from oslo_db.sqlalchemy import test_base
23 23
 from oslo_db.sqlalchemy import test_migrations
24 24
 import sqlalchemy.types as types
25 25
 
26
-from glance.db import migration as dm
26
+from glance.db import migration as db_migration
27 27
 from glance.db.sqlalchemy import alembic_migrations
28 28
 from glance.db.sqlalchemy.alembic_migrations import versions
29 29
 from glance.db.sqlalchemy import models
@@ -34,10 +34,11 @@ import glance.tests.utils as test_utils
34 34
 
35 35
 class AlembicMigrationsMixin(object):
36 36
 
37
-    def _get_revisions(self, config):
37
+    def _get_revisions(self, config, head=None):
38
+        head = head or db_migration.LATEST_REVISION
38 39
         scripts_dir = alembic_script.ScriptDirectory.from_config(config)
39 40
         revisions = list(scripts_dir.walk_revisions(base='base',
40
-                                                    head=dm.LATEST_REVISION))
41
+                                                    head=head))
41 42
         revisions = list(reversed(revisions))
42 43
         revisions = [rev.revision for rev in revisions]
43 44
         return revisions

Loading…
Cancel
Save