Browse Source

Port Glance Migrations to Alembic

This change proposes the use of Alembic to manage Glance migrations.
* Introduce new directory ``alembic_migrations`` under
  ``glance/db/sqlalchemy``. This directory is the home for all glance
  migrations henceforth. All the migration scripts reside under
  ``versions`` directory.
* All the migrations up to Liberty are consolidated into one migration
  called ``liberty_initial`` as those migrations are not supported
  any more. Mitaka migrations are retained but under a different naming
  convention.
* All the glance manage db commands are changed appropriately. They now
  use alembic to perform operations such as ``version``, ``upgrade``,
  ``sync`` and ``version_control``.
* The database versions are not numerical any more. They are the revision
  ID of the last migration applied on the database. Since we don't
  support migrations before Mitaka, the Liberty version ``42`` will now
  appear as ``liberty``. Migration ``43`` and ``44`` in Mitaka appear as
  ``mitaka01`` and ``mitaka02`` respectively.
* When one performs a ``sync`` or ``upgrade`` command, the database is
  first stamped with an equivalent alembic version before upgrading.
* The older migration scripts are retained so that users can correlate
  with the new migrations. Also, it is probably safe to retain them until
  the alembic migrations become stable. Similarly, the ``migrate_version``
  table is not removed yet.

Partially-Implements: blueprint alembic-migrations

Change-Id: Ie8594ff339a13bf190aefa308f54e97ee20ecfa2
Co-Authored-By: Alexander Bashmakov <alexander.bashmakov@intel.com>
Depends-On: I1596499529af249bc48dfe859bbd31e90c48a5e0
tags/14.0.0.0rc1
Hemanth Makkapati 2 years ago
parent
commit
21d431013f

+ 3
- 2
doc/source/db.rst View File

@@ -29,9 +29,10 @@ The commands should be executed as a subcommand of 'db':
29 29
 Sync the Database
30 30
 -----------------
31 31
 
32
-    glance-manage db sync <version> <current_version>
32
+    glance-manage db sync <VERSION>
33 33
 
34
-Place a database under migration control and upgrade, creating it first if necessary.
34
+Place an existing database under migration control and upgrade it to the
35
+specified VERSION.
35 36
 
36 37
 
37 38
 Determining the Database Version

+ 3
- 7
doc/source/man/glancemanage.rst View File

@@ -53,9 +53,9 @@ COMMANDS
53 53
   **db_version_control**
54 54
         Place the database under migration control.
55 55
 
56
-  **db_sync <VERSION> <CURRENT_VERSION>**
57
-        Place a database under migration control and upgrade, creating
58
-        it first if necessary.
56
+  **db_sync <VERSION>**
57
+        Place an existing database under migration control and upgrade it to
58
+        the specified VERSION.
59 59
 
60 60
   **db_export_metadefs [PATH | PREFIX]**
61 61
         Export the metadata definitions into json format. By default the
@@ -80,10 +80,6 @@ OPTIONS
80 80
 
81 81
   .. include:: general_options.rst
82 82
 
83
-  **--sql_connection=CONN_STRING**
84
-        A proper SQLAlchemy connection string as described
85
-        `here <http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html?highlight=engine#sqlalchemy.create_engine>`_
86
-
87 83
 .. include:: footer.rst
88 84
 
89 85
 CONFIGURATION

+ 47
- 30
glance/cmd/manage.py View File

@@ -39,8 +39,9 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
39 39
 if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
40 40
     sys.path.insert(0, possible_topdir)
41 41
 
42
+from alembic import command as alembic_command
43
+
42 44
 from oslo_config import cfg
43
-from oslo_db.sqlalchemy import migration
44 45
 from oslo_log import log as logging
45 46
 from oslo_utils import encodeutils
46 47
 import six
@@ -49,6 +50,7 @@ from glance.common import config
49 50
 from glance.common import exception
50 51
 from glance import context
51 52
 from glance.db import migration as db_migration
53
+from glance.db.sqlalchemy import alembic_migrations
52 54
 from glance.db.sqlalchemy import api as db_api
53 55
 from glance.db.sqlalchemy import metadata
54 56
 from glance.i18n import _
@@ -73,39 +75,56 @@ class DbCommands(object):
73 75
 
74 76
     def version(self):
75 77
         """Print database's current migration level"""
76
-        print(migration.db_version(db_api.get_engine(),
77
-                                   db_migration.MIGRATE_REPO_PATH,
78
-                                   db_migration.INIT_VERSION))
78
+        current_heads = alembic_migrations.get_current_alembic_heads()
79
+        if current_heads:
80
+            # Migrations are managed by alembic
81
+            for head in current_heads:
82
+                print(head)
83
+        else:
84
+            # Migrations are managed by legacy versioning scheme
85
+            print(_('Database is either not under migration control or under '
86
+                    'legacy migration control, please run '
87
+                    '"glance-manage db sync" to place the database under '
88
+                    'alembic migration control.'))
79 89
 
80 90
     @args('--version', metavar='<version>', help='Database version')
81
-    def upgrade(self, version=None):
91
+    def upgrade(self, version='heads'):
82 92
         """Upgrade the database's migration level"""
83
-        migration.db_sync(db_api.get_engine(),
84
-                          db_migration.MIGRATE_REPO_PATH,
85
-                          version)
93
+        self.sync(version)
86 94
 
87 95
     @args('--version', metavar='<version>', help='Database version')
88
-    def version_control(self, version=None):
96
+    def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION):
89 97
         """Place a database under migration control"""
90
-        migration.db_version_control(db_api.get_engine(),
91
-                                     db_migration.MIGRATE_REPO_PATH,
92
-                                     version)
98
+
99
+        if version is None:
100
+            version = db_migration.ALEMBIC_INIT_VERSION
101
+
102
+        a_config = alembic_migrations.get_alembic_config()
103
+        alembic_command.stamp(a_config, version)
104
+        print(_("Placed database under migration control at "
105
+                "revision:"), version)
93 106
 
94 107
     @args('--version', metavar='<version>', help='Database version')
95
-    @args('--current_version', metavar='<version>',
96
-          help='Current Database version')
97
-    def sync(self, version=None, current_version=None):
108
+    def sync(self, version='heads'):
98 109
         """
99
-        Place a database under migration control and upgrade it,
100
-        creating first if necessary.
110
+        Place an existing database under migration control and upgrade it.
101 111
         """
102
-        if current_version not in (None, 'None'):
103
-            migration.db_version_control(db_api.get_engine(),
104
-                                         db_migration.MIGRATE_REPO_PATH,
105
-                                         version=current_version)
106
-        migration.db_sync(db_api.get_engine(),
107
-                          db_migration.MIGRATE_REPO_PATH,
108
-                          version)
112
+        if version is None:
113
+            version = 'heads'
114
+
115
+        alembic_migrations.place_database_under_alembic_control()
116
+
117
+        a_config = alembic_migrations.get_alembic_config()
118
+        alembic_command.upgrade(a_config, version)
119
+        heads = alembic_migrations.get_current_alembic_heads()
120
+        if heads is None:
121
+            raise Exception("Database sync failed")
122
+        revs = ", ".join(heads)
123
+        if version is 'heads':
124
+            print(_("Upgraded database, current revision(s):"), revs)
125
+        else:
126
+            print(_('Upgraded database to: %(v)s, current revision(s): %(r)s')
127
+                  % {'v': version, 'r': revs})
109 128
 
110 129
     @args('--path', metavar='<path>', help='Path to the directory or file '
111 130
                                            'where json metadata is stored')
@@ -179,15 +198,14 @@ class DbLegacyCommands(object):
179 198
     def version(self):
180 199
         self.command_object.version()
181 200
 
182
-    def upgrade(self, version=None):
201
+    def upgrade(self, version='heads'):
183 202
         self.command_object.upgrade(CONF.command.version)
184 203
 
185
-    def version_control(self, version=None):
204
+    def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION):
186 205
         self.command_object.version_control(CONF.command.version)
187 206
 
188
-    def sync(self, version=None, current_version=None):
189
-        self.command_object.sync(CONF.command.version,
190
-                                 CONF.command.current_version)
207
+    def sync(self, version='heads'):
208
+        self.command_object.sync(CONF.command.version)
191 209
 
192 210
     def load_metadefs(self, path=None, merge=False,
193 211
                       prefer_new=False, overwrite=False):
@@ -224,7 +242,6 @@ def add_legacy_command_parsers(command_object, subparsers):
224 242
     parser = subparsers.add_parser('db_sync')
225 243
     parser.set_defaults(action_fn=legacy_command_object.sync)
226 244
     parser.add_argument('version', nargs='?')
227
-    parser.add_argument('current_version', nargs='?')
228 245
     parser.set_defaults(action='db_sync')
229 246
 
230 247
     parser = subparsers.add_parser('db_load_metadefs')

+ 1
- 0
glance/db/migration.py View File

@@ -45,6 +45,7 @@ def get_backend():
45 45
                     cfg.CONF.database.backend).driver
46 46
     return _IMPL
47 47
 
48
+ALEMBIC_INIT_VERSION = 'liberty'
48 49
 INIT_VERSION = 0
49 50
 
50 51
 MIGRATE_REPO_PATH = os.path.join(

+ 1
- 0
glance/db/sqlalchemy/alembic_migrations/README View File

@@ -0,0 +1 @@
1
+Generic single-database configuration.

+ 99
- 0
glance/db/sqlalchemy/alembic_migrations/__init__.py View File

@@ -0,0 +1,99 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2013 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+import os
17
+import sys
18
+
19
+from alembic import command as alembic_command
20
+from alembic import config as alembic_config
21
+from alembic import migration as alembic_migration
22
+from oslo_db import exception as db_exception
23
+from oslo_db.sqlalchemy import migration
24
+
25
+from glance.db import migration as db_migration
26
+from glance.db.sqlalchemy import api as db_api
27
+from glance.i18n import _
28
+
29
+
30
+def get_alembic_config():
31
+    """Return a valid alembic config object"""
32
+    ini_path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
33
+    config = alembic_config.Config(os.path.abspath(ini_path))
34
+    dbconn = str(db_api.get_engine().url)
35
+    config.set_main_option('sqlalchemy.url', dbconn)
36
+    return config
37
+
38
+
39
+def get_current_alembic_heads():
40
+    """Return current heads (if any) from the alembic migration table"""
41
+    engine = db_api.get_engine()
42
+    with engine.connect() as conn:
43
+        context = alembic_migration.MigrationContext.configure(conn)
44
+        heads = context.get_current_heads()
45
+        return heads
46
+
47
+
48
+def get_current_legacy_head():
49
+    try:
50
+        legacy_head = migration.db_version(db_api.get_engine(),
51
+                                           db_migration.MIGRATE_REPO_PATH,
52
+                                           db_migration.INIT_VERSION)
53
+    except db_exception.DbMigrationError:
54
+        legacy_head = None
55
+    return legacy_head
56
+
57
+
58
+def is_database_under_alembic_control():
59
+    if get_current_alembic_heads():
60
+        return True
61
+    return False
62
+
63
+
64
+def is_database_under_migrate_control():
65
+    if get_current_legacy_head():
66
+        return True
67
+    return False
68
+
69
+
70
+def place_database_under_alembic_control():
71
+    a_config = get_alembic_config()
72
+
73
+    if not is_database_under_migrate_control():
74
+        return
75
+
76
+    if not is_database_under_alembic_control():
77
+        print(_("Database is currently not under Alembic's migration "
78
+                "control."))
79
+        head = get_current_legacy_head()
80
+        if head == 42:
81
+            alembic_version = 'liberty'
82
+        elif head == 43:
83
+            alembic_version = 'mitaka01'
84
+        elif head == 44:
85
+            alembic_version = 'mitaka02'
86
+        elif head == 45:
87
+            alembic_version = 'ocata01'
88
+        elif head in range(1, 42):
89
+            print("Legacy head: ", head)
90
+            sys.exit(_("The current database version is not supported any "
91
+                       "more. Please upgrade to Liberty release first."))
92
+        else:
93
+            sys.exit(_("Unable to place database under Alembic's migration "
94
+                       "control. Unknown database state, can't proceed "
95
+                       "further."))
96
+
97
+        print(_("Placing database under Alembic's migration control at "
98
+                "revision:"), alembic_version)
99
+        alembic_command.stamp(a_config, alembic_version)

+ 224
- 0
glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py View File

@@ -0,0 +1,224 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2013 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+from alembic import op
17
+from sqlalchemy.schema import (
18
+    Column, PrimaryKeyConstraint, ForeignKeyConstraint)
19
+
20
+from glance.db.sqlalchemy.migrate_repo.schema import (
21
+    Boolean, DateTime, Integer, BigInteger, String, Text, Numeric)  # noqa
22
+
23
+
24
+def _add_artifacts_table():
25
+    op.create_table('artifacts',
26
+                    Column('id', String(length=36), nullable=False),
27
+                    Column('name', String(length=255), nullable=False),
28
+                    Column('type_name', String(length=255), nullable=False),
29
+                    Column('type_version_prefix',
30
+                           BigInteger(),
31
+                           nullable=False),
32
+                    Column('type_version_suffix',
33
+                           String(length=255),
34
+                           nullable=True),
35
+                    Column('type_version_meta',
36
+                           String(length=255),
37
+                           nullable=True),
38
+                    Column('version_prefix', BigInteger(), nullable=False),
39
+                    Column('version_suffix',
40
+                           String(length=255),
41
+                           nullable=True),
42
+                    Column('version_meta', String(length=255), nullable=True),
43
+                    Column('description', Text(), nullable=True),
44
+                    Column('visibility', String(length=32), nullable=False),
45
+                    Column('state', String(length=32), nullable=False),
46
+                    Column('owner', String(length=255), nullable=False),
47
+                    Column('created_at', DateTime(), nullable=False),
48
+                    Column('updated_at', DateTime(), nullable=False),
49
+                    Column('deleted_at', DateTime(), nullable=True),
50
+                    Column('published_at', DateTime(), nullable=True),
51
+                    PrimaryKeyConstraint('id'),
52
+                    mysql_engine='InnoDB',
53
+                    mysql_charset='utf8',
54
+                    extend_existing=True)
55
+
56
+    op.create_index('ix_artifact_name_and_version',
57
+                    'artifacts',
58
+                    ['name', 'version_prefix', 'version_suffix'],
59
+                    unique=False)
60
+    op.create_index('ix_artifact_owner', 'artifacts', ['owner'], unique=False)
61
+    op.create_index('ix_artifact_state', 'artifacts', ['state'], unique=False)
62
+    op.create_index('ix_artifact_type',
63
+                    'artifacts',
64
+                    ['type_name',
65
+                     'type_version_prefix',
66
+                     'type_version_suffix'],
67
+                    unique=False)
68
+    op.create_index('ix_artifact_visibility',
69
+                    'artifacts',
70
+                    ['visibility'],
71
+                    unique=False)
72
+
73
+
74
+def _add_artifact_blobs_table():
75
+    op.create_table('artifact_blobs',
76
+                    Column('id', String(length=36), nullable=False),
77
+                    Column('artifact_id', String(length=36), nullable=False),
78
+                    Column('size', BigInteger(), nullable=False),
79
+                    Column('checksum', String(length=32), nullable=True),
80
+                    Column('name', String(length=255), nullable=False),
81
+                    Column('item_key', String(length=329), nullable=True),
82
+                    Column('position', Integer(), nullable=True),
83
+                    Column('created_at', DateTime(), nullable=False),
84
+                    Column('updated_at', DateTime(), nullable=False),
85
+                    ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ),
86
+                    PrimaryKeyConstraint('id'),
87
+                    mysql_engine='InnoDB',
88
+                    mysql_charset='utf8',
89
+                    extend_existing=True)
90
+
91
+    op.create_index('ix_artifact_blobs_artifact_id',
92
+                    'artifact_blobs',
93
+                    ['artifact_id'],
94
+                    unique=False)
95
+    op.create_index('ix_artifact_blobs_name',
96
+                    'artifact_blobs',
97
+                    ['name'],
98
+                    unique=False)
99
+
100
+
101
+def _add_artifact_dependencies_table():
102
+    op.create_table('artifact_dependencies',
103
+                    Column('id', String(length=36), nullable=False),
104
+                    Column('artifact_source',
105
+                           String(length=36),
106
+                           nullable=False),
107
+                    Column('artifact_dest', String(length=36), nullable=False),
108
+                    Column('artifact_origin',
109
+                           String(length=36),
110
+                           nullable=False),
111
+                    Column('is_direct', Boolean(), nullable=False),
112
+                    Column('position', Integer(), nullable=True),
113
+                    Column('name', String(length=36), nullable=True),
114
+                    Column('created_at', DateTime(), nullable=False),
115
+                    Column('updated_at', DateTime(), nullable=False),
116
+                    ForeignKeyConstraint(['artifact_dest'],
117
+                                         ['artifacts.id'], ),
118
+                    ForeignKeyConstraint(['artifact_origin'],
119
+                                         ['artifacts.id'], ),
120
+                    ForeignKeyConstraint(['artifact_source'],
121
+                                         ['artifacts.id'], ),
122
+                    PrimaryKeyConstraint('id'),
123
+                    mysql_engine='InnoDB',
124
+                    mysql_charset='utf8',
125
+                    extend_existing=True)
126
+
127
+    op.create_index('ix_artifact_dependencies_dest_id',
128
+                    'artifact_dependencies',
129
+                    ['artifact_dest'],
130
+                    unique=False)
131
+    op.create_index('ix_artifact_dependencies_direct_dependencies',
132
+                    'artifact_dependencies',
133
+                    ['artifact_source', 'is_direct'],
134
+                    unique=False)
135
+    op.create_index('ix_artifact_dependencies_origin_id',
136
+                    'artifact_dependencies',
137
+                    ['artifact_origin'],
138
+                    unique=False)
139
+    op.create_index('ix_artifact_dependencies_source_id',
140
+                    'artifact_dependencies',
141
+                    ['artifact_source'],
142
+                    unique=False)
143
+
144
+
145
+def _add_artifact_properties_table():
146
+    op.create_table('artifact_properties',
147
+                    Column('id', String(length=36), nullable=False),
148
+                    Column('artifact_id', String(length=36), nullable=False),
149
+                    Column('name', String(length=255), nullable=False),
150
+                    Column('string_value', String(length=255), nullable=True),
151
+                    Column('int_value', Integer(), nullable=True),
152
+                    Column('numeric_value', Numeric(), nullable=True),
153
+                    Column('bool_value', Boolean(), nullable=True),
154
+                    Column('text_value', Text(), nullable=True),
155
+                    Column('created_at', DateTime(), nullable=False),
156
+                    Column('updated_at', DateTime(), nullable=False),
157
+                    Column('position', Integer(), nullable=True),
158
+                    ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ),
159
+                    PrimaryKeyConstraint('id'),
160
+                    mysql_engine='InnoDB',
161
+                    mysql_charset='utf8',
162
+                    extend_existing=True)
163
+
164
+    op.create_index('ix_artifact_properties_artifact_id',
165
+                    'artifact_properties',
166
+                    ['artifact_id'],
167
+                    unique=False)
168
+    op.create_index('ix_artifact_properties_name',
169
+                    'artifact_properties',
170
+                    ['name'],
171
+                    unique=False)
172
+
173
+
174
+def _add_artifact_tags_table():
175
+    op.create_table('artifact_tags',
176
+                    Column('id', String(length=36), nullable=False),
177
+                    Column('artifact_id', String(length=36), nullable=False),
178
+                    Column('value', String(length=255), nullable=False),
179
+                    Column('created_at', DateTime(), nullable=False),
180
+                    Column('updated_at', DateTime(), nullable=False),
181
+                    ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ),
182
+                    PrimaryKeyConstraint('id'),
183
+                    mysql_engine='InnoDB',
184
+                    mysql_charset='utf8',
185
+                    extend_existing=True)
186
+
187
+    op.create_index('ix_artifact_tags_artifact_id',
188
+                    'artifact_tags',
189
+                    ['artifact_id'],
190
+                    unique=False)
191
+    op.create_index('ix_artifact_tags_artifact_id_tag_value',
192
+                    'artifact_tags',
193
+                    ['artifact_id', 'value'],
194
+                    unique=False)
195
+
196
+
197
+def _add_artifact_blob_locations_table():
198
+    op.create_table('artifact_blob_locations',
199
+                    Column('id', String(length=36), nullable=False),
200
+                    Column('blob_id', String(length=36), nullable=False),
201
+                    Column('value', Text(), nullable=False),
202
+                    Column('created_at', DateTime(), nullable=False),
203
+                    Column('updated_at', DateTime(), nullable=False),
204
+                    Column('position', Integer(), nullable=True),
205
+                    Column('status', String(length=36), nullable=True),
206
+                    ForeignKeyConstraint(['blob_id'], ['artifact_blobs.id'], ),
207
+                    PrimaryKeyConstraint('id'),
208
+                    mysql_engine='InnoDB',
209
+                    mysql_charset='utf8',
210
+                    extend_existing=True)
211
+
212
+    op.create_index('ix_artifact_blob_locations_blob_id',
213
+                    'artifact_blob_locations',
214
+                    ['blob_id'],
215
+                    unique=False)
216
+
217
+
218
+def upgrade():
219
+    _add_artifacts_table()
220
+    _add_artifact_blobs_table()
221
+    _add_artifact_dependencies_table()
222
+    _add_artifact_properties_table()
223
+    _add_artifact_tags_table()
224
+    _add_artifact_blob_locations_table()

+ 201
- 0
glance/db/sqlalchemy/alembic_migrations/add_images_tables.py View File

@@ -0,0 +1,201 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2013 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+from alembic import op
17
+from sqlalchemy import sql
18
+from sqlalchemy.schema import (
19
+    Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint)
20
+
21
+from glance.db.sqlalchemy.migrate_repo.schema import (
22
+    Boolean, DateTime, Integer, BigInteger, String, Text)  # noqa
23
+from glance.db.sqlalchemy.models import JSONEncodedDict
24
+
25
+
26
+def _add_images_table():
27
+    op.create_table('images',
28
+                    Column('id', String(length=36), nullable=False),
29
+                    Column('name', String(length=255), nullable=True),
30
+                    Column('size', BigInteger(), nullable=True),
31
+                    Column('status', String(length=30), nullable=False),
32
+                    Column('is_public', Boolean(), nullable=False),
33
+                    Column('created_at', DateTime(), nullable=False),
34
+                    Column('updated_at', DateTime(), nullable=True),
35
+                    Column('deleted_at', DateTime(), nullable=True),
36
+                    Column('deleted', Boolean(), nullable=False),
37
+                    Column('disk_format', String(length=20), nullable=True),
38
+                    Column('container_format',
39
+                           String(length=20),
40
+                           nullable=True),
41
+                    Column('checksum', String(length=32), nullable=True),
42
+                    Column('owner', String(length=255), nullable=True),
43
+                    Column('min_disk', Integer(), nullable=False),
44
+                    Column('min_ram', Integer(), nullable=False),
45
+                    Column('protected',
46
+                           Boolean(),
47
+                           server_default=sql.false(),
48
+                           nullable=False),
49
+                    Column('virtual_size', BigInteger(), nullable=True),
50
+                    PrimaryKeyConstraint('id'),
51
+                    mysql_engine='InnoDB',
52
+                    mysql_charset='utf8',
53
+                    extend_existing=True)
54
+
55
+    op.create_index('checksum_image_idx',
56
+                    'images',
57
+                    ['checksum'],
58
+                    unique=False)
59
+    op.create_index('ix_images_deleted',
60
+                    'images',
61
+                    ['deleted'],
62
+                    unique=False)
63
+    op.create_index('ix_images_is_public',
64
+                    'images',
65
+                    ['is_public'],
66
+                    unique=False)
67
+    op.create_index('owner_image_idx',
68
+                    'images',
69
+                    ['owner'],
70
+                    unique=False)
71
+
72
+
73
+def _add_image_properties_table():
74
+    op.create_table('image_properties',
75
+                    Column('id', Integer(), nullable=False),
76
+                    Column('image_id', String(length=36), nullable=False),
77
+                    Column('name', String(length=255), nullable=False),
78
+                    Column('value', Text(), nullable=True),
79
+                    Column('created_at', DateTime(), nullable=False),
80
+                    Column('updated_at', DateTime(), nullable=True),
81
+                    Column('deleted_at', DateTime(), nullable=True),
82
+                    Column('deleted', Boolean(), nullable=False),
83
+                    PrimaryKeyConstraint('id'),
84
+                    ForeignKeyConstraint(['image_id'], ['images.id'], ),
85
+                    UniqueConstraint('image_id',
86
+                                     'name',
87
+                                     name='ix_image_properties_image_id_name'),
88
+                    mysql_engine='InnoDB',
89
+                    mysql_charset='utf8',
90
+                    extend_existing=True)
91
+
92
+    op.create_index('ix_image_properties_deleted',
93
+                    'image_properties',
94
+                    ['deleted'],
95
+                    unique=False)
96
+    op.create_index('ix_image_properties_image_id',
97
+                    'image_properties',
98
+                    ['image_id'],
99
+                    unique=False)
100
+
101
+
102
+def _add_image_locations_table():
103
+    op.create_table('image_locations',
104
+                    Column('id', Integer(), nullable=False),
105
+                    Column('image_id', String(length=36), nullable=False),
106
+                    Column('value', Text(), nullable=False),
107
+                    Column('created_at', DateTime(), nullable=False),
108
+                    Column('updated_at', DateTime(), nullable=True),
109
+                    Column('deleted_at', DateTime(), nullable=True),
110
+                    Column('deleted', Boolean(), nullable=False),
111
+                    Column('meta_data', JSONEncodedDict(), nullable=True),
112
+                    Column('status',
113
+                           String(length=30),
114
+                           server_default='active',
115
+                           nullable=False),
116
+                    PrimaryKeyConstraint('id'),
117
+                    ForeignKeyConstraint(['image_id'], ['images.id'], ),
118
+                    mysql_engine='InnoDB',
119
+                    mysql_charset='utf8',
120
+                    extend_existing=True)
121
+
122
+    op.create_index('ix_image_locations_deleted',
123
+                    'image_locations',
124
+                    ['deleted'],
125
+                    unique=False)
126
+    op.create_index('ix_image_locations_image_id',
127
+                    'image_locations',
128
+                    ['image_id'],
129
+                    unique=False)
130
+
131
+
132
+def _add_image_members_table():
133
+    deleted_member_constraint = 'image_members_image_id_member_deleted_at_key'
134
+    op.create_table('image_members',
135
+                    Column('id', Integer(), nullable=False),
136
+                    Column('image_id', String(length=36), nullable=False),
137
+                    Column('member', String(length=255), nullable=False),
138
+                    Column('can_share', Boolean(), nullable=False),
139
+                    Column('created_at', DateTime(), nullable=False),
140
+                    Column('updated_at', DateTime(), nullable=True),
141
+                    Column('deleted_at', DateTime(), nullable=True),
142
+                    Column('deleted', Boolean(), nullable=False),
143
+                    Column('status',
144
+                           String(length=20),
145
+                           server_default='pending',
146
+                           nullable=False),
147
+                    ForeignKeyConstraint(['image_id'], ['images.id'], ),
148
+                    PrimaryKeyConstraint('id'),
149
+                    UniqueConstraint('image_id',
150
+                                     'member',
151
+                                     'deleted_at',
152
+                                     name=deleted_member_constraint),
153
+                    mysql_engine='InnoDB',
154
+                    mysql_charset='utf8',
155
+                    extend_existing=True)
156
+
157
+    op.create_index('ix_image_members_deleted',
158
+                    'image_members',
159
+                    ['deleted'],
160
+                    unique=False)
161
+    op.create_index('ix_image_members_image_id',
162
+                    'image_members',
163
+                    ['image_id'],
164
+                    unique=False)
165
+    op.create_index('ix_image_members_image_id_member',
166
+                    'image_members',
167
+                    ['image_id', 'member'],
168
+                    unique=False)
169
+
170
+
171
+def _add_images_tags_table():
172
+    op.create_table('image_tags',
173
+                    Column('id', Integer(), nullable=False),
174
+                    Column('image_id', String(length=36), nullable=False),
175
+                    Column('value', String(length=255), nullable=False),
176
+                    Column('created_at', DateTime(), nullable=False),
177
+                    Column('updated_at', DateTime(), nullable=True),
178
+                    Column('deleted_at', DateTime(), nullable=True),
179
+                    Column('deleted', Boolean(), nullable=False),
180
+                    ForeignKeyConstraint(['image_id'], ['images.id'], ),
181
+                    PrimaryKeyConstraint('id'),
182
+                    mysql_engine='InnoDB',
183
+                    mysql_charset='utf8',
184
+                    extend_existing=True)
185
+
186
+    op.create_index('ix_image_tags_image_id',
187
+                    'image_tags',
188
+                    ['image_id'],
189
+                    unique=False)
190
+    op.create_index('ix_image_tags_image_id_tag_value',
191
+                    'image_tags',
192
+                    ['image_id', 'value'],
193
+                    unique=False)
194
+
195
+
196
+def upgrade():
197
+    _add_images_table()
198
+    _add_image_properties_table()
199
+    _add_image_locations_table()
200
+    _add_image_members_table()
201
+    _add_images_tags_table()

+ 171
- 0
glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py View File

@@ -0,0 +1,171 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2013 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+from alembic import op
17
+from sqlalchemy.schema import (
18
+    Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint)
19
+
20
+from glance.db.sqlalchemy.migrate_repo.schema import (
21
+    Boolean, DateTime, Integer, String, Text)  # noqa
22
+from glance.db.sqlalchemy.models import JSONEncodedDict
23
+
24
+
25
+def _add_metadef_namespaces_table():
26
+    op.create_table('metadef_namespaces',
27
+                    Column('id', Integer(), nullable=False),
28
+                    Column('namespace', String(length=80), nullable=False),
29
+                    Column('display_name', String(length=80), nullable=True),
30
+                    Column('description', Text(), nullable=True),
31
+                    Column('visibility', String(length=32), nullable=True),
32
+                    Column('protected', Boolean(), nullable=True),
33
+                    Column('owner', String(length=255), nullable=False),
34
+                    Column('created_at', DateTime(), nullable=False),
35
+                    Column('updated_at', DateTime(), nullable=True),
36
+                    PrimaryKeyConstraint('id'),
37
+                    UniqueConstraint('namespace',
38
+                                     name='uq_metadef_namespaces_namespace'),
39
+                    mysql_engine='InnoDB',
40
+                    mysql_charset='utf8',
41
+                    extend_existing=True)
42
+
43
+    op.create_index('ix_metadef_namespaces_owner',
44
+                    'metadef_namespaces',
45
+                    ['owner'],
46
+                    unique=False)
47
+
48
+
49
+def _add_metadef_resource_types_table():
50
+    op.create_table('metadef_resource_types',
51
+                    Column('id', Integer(), nullable=False),
52
+                    Column('name', String(length=80), nullable=False),
53
+                    Column('protected', Boolean(), nullable=False),
54
+                    Column('created_at', DateTime(), nullable=False),
55
+                    Column('updated_at', DateTime(), nullable=True),
56
+                    PrimaryKeyConstraint('id'),
57
+                    UniqueConstraint('name',
58
+                                     name='uq_metadef_resource_types_name'),
59
+                    mysql_engine='InnoDB',
60
+                    mysql_charset='utf8',
61
+                    extend_existing=True)
62
+
63
+
64
+def _add_metadef_namespace_resource_types_table():
65
+    op.create_table('metadef_namespace_resource_types',
66
+                    Column('resource_type_id', Integer(), nullable=False),
67
+                    Column('namespace_id', Integer(), nullable=False),
68
+                    Column('properties_target',
69
+                           String(length=80),
70
+                           nullable=True),
71
+                    Column('prefix', String(length=80), nullable=True),
72
+                    Column('created_at', DateTime(), nullable=False),
73
+                    Column('updated_at', DateTime(), nullable=True),
74
+                    ForeignKeyConstraint(['namespace_id'],
75
+                                         ['metadef_namespaces.id'], ),
76
+                    ForeignKeyConstraint(['resource_type_id'],
77
+                                         ['metadef_resource_types.id'], ),
78
+                    PrimaryKeyConstraint('resource_type_id', 'namespace_id'),
79
+                    mysql_engine='InnoDB',
80
+                    mysql_charset='utf8',
81
+                    extend_existing=True)
82
+
83
+    op.create_index('ix_metadef_ns_res_types_namespace_id',
84
+                    'metadef_namespace_resource_types',
85
+                    ['namespace_id'],
86
+                    unique=False)
87
+
88
+
89
+def _add_metadef_objects_table():
90
+    ns_id_name_constraint = 'uq_metadef_objects_namespace_id_name'
91
+
92
+    op.create_table('metadef_objects',
93
+                    Column('id', Integer(), nullable=False),
94
+                    Column('namespace_id', Integer(), nullable=False),
95
+                    Column('name', String(length=80), nullable=False),
96
+                    Column('description', Text(), nullable=True),
97
+                    Column('required', Text(), nullable=True),
98
+                    Column('json_schema', JSONEncodedDict(), nullable=False),
99
+                    Column('created_at', DateTime(), nullable=False),
100
+                    Column('updated_at', DateTime(), nullable=True),
101
+                    ForeignKeyConstraint(['namespace_id'],
102
+                                         ['metadef_namespaces.id'], ),
103
+                    PrimaryKeyConstraint('id'),
104
+                    UniqueConstraint('namespace_id',
105
+                                     'name',
106
+                                     name=ns_id_name_constraint),
107
+                    mysql_engine='InnoDB',
108
+                    mysql_charset='utf8',
109
+                    extend_existing=True)
110
+
111
+    op.create_index('ix_metadef_objects_name',
112
+                    'metadef_objects',
113
+                    ['name'],
114
+                    unique=False)
115
+
116
+
117
+def _add_metadef_properties_table():
118
+    ns_id_name_constraint = 'uq_metadef_properties_namespace_id_name'
119
+    op.create_table('metadef_properties',
120
+                    Column('id', Integer(), nullable=False),
121
+                    Column('namespace_id', Integer(), nullable=False),
122
+                    Column('name', String(length=80), nullable=False),
123
+                    Column('json_schema', JSONEncodedDict(), nullable=False),
124
+                    Column('created_at', DateTime(), nullable=False),
125
+                    Column('updated_at', DateTime(), nullable=True),
126
+                    ForeignKeyConstraint(['namespace_id'],
127
+                                         ['metadef_namespaces.id'], ),
128
+                    PrimaryKeyConstraint('id'),
129
+                    UniqueConstraint('namespace_id',
130
+                                     'name',
131
+                                     name=ns_id_name_constraint),
132
+                    mysql_engine='InnoDB',
133
+                    mysql_charset='utf8',
134
+                    extend_existing=True)
135
+
136
+    op.create_index('ix_metadef_properties_name',
137
+                    'metadef_properties',
138
+                    ['name'],
139
+                    unique=False)
140
+
141
+
142
+def _add_metadef_tags_table():
143
+    op.create_table('metadef_tags',
144
+                    Column('id', Integer(), nullable=False),
145
+                    Column('namespace_id', Integer(), nullable=False),
146
+                    Column('name', String(length=80), nullable=False),
147
+                    Column('created_at', DateTime(), nullable=False),
148
+                    Column('updated_at', DateTime(), nullable=True),
149
+                    ForeignKeyConstraint(['namespace_id'],
150
+                                         ['metadef_namespaces.id'], ),
151
+                    PrimaryKeyConstraint('id'),
152
+                    UniqueConstraint('namespace_id',
153
+                                     'name',
154
+                                     name='uq_metadef_tags_namespace_id_name'),
155
+                    mysql_engine='InnoDB',
156
+                    mysql_charset='utf8',
157
+                    extend_existing=True)
158
+
159
+    op.create_index('ix_metadef_tags_name',
160
+                    'metadef_tags',
161
+                    ['name'],
162
+                    unique=False)
163
+
164
+
165
+def upgrade():
166
+    _add_metadef_namespaces_table()
167
+    _add_metadef_resource_types_table()
168
+    _add_metadef_namespace_resource_types_table()
169
+    _add_metadef_objects_table()
170
+    _add_metadef_properties_table()
171
+    _add_metadef_tags_table()

+ 66
- 0
glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py View File

@@ -0,0 +1,66 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2013 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+from alembic import op
17
+from sqlalchemy.schema import (
18
+    Column, PrimaryKeyConstraint, ForeignKeyConstraint)
19
+
20
+from glance.db.sqlalchemy.migrate_repo.schema import (
21
+    Boolean, DateTime, String, Text)  # noqa
22
+from glance.db.sqlalchemy.models import JSONEncodedDict
23
+
24
+
25
+def _add_tasks_table():
26
+    op.create_table('tasks',
27
+                    Column('id', String(length=36), nullable=False),
28
+                    Column('type', String(length=30), nullable=False),
29
+                    Column('status', String(length=30), nullable=False),
30
+                    Column('owner', String(length=255), nullable=False),
31
+                    Column('expires_at', DateTime(), nullable=True),
32
+                    Column('created_at', DateTime(), nullable=False),
33
+                    Column('updated_at', DateTime(), nullable=True),
34
+                    Column('deleted_at', DateTime(), nullable=True),
35
+                    Column('deleted', Boolean(), nullable=False),
36
+                    PrimaryKeyConstraint('id'),
37
+                    mysql_engine='InnoDB',
38
+                    mysql_charset='utf8',
39
+                    extend_existing=True)
40
+
41
+    op.create_index('ix_tasks_deleted', 'tasks', ['deleted'], unique=False)
42
+    op.create_index('ix_tasks_owner', 'tasks', ['owner'], unique=False)
43
+    op.create_index('ix_tasks_status', 'tasks', ['status'], unique=False)
44
+    op.create_index('ix_tasks_type', 'tasks', ['type'], unique=False)
45
+    op.create_index('ix_tasks_updated_at',
46
+                    'tasks',
47
+                    ['updated_at'],
48
+                    unique=False)
49
+
50
+
51
+def _add_task_info_table():
52
+    op.create_table('task_info',
53
+                    Column('task_id', String(length=36), nullable=False),
54
+                    Column('input', JSONEncodedDict(), nullable=True),
55
+                    Column('result', JSONEncodedDict(), nullable=True),
56
+                    Column('message', Text(), nullable=True),
57
+                    ForeignKeyConstraint(['task_id'], ['tasks.id'], ),
58
+                    PrimaryKeyConstraint('task_id'),
59
+                    mysql_engine='InnoDB',
60
+                    mysql_charset='utf8',
61
+                    extend_existing=True)
62
+
63
+
64
+def upgrade():
65
+    _add_tasks_table()
66
+    _add_task_info_table()

+ 69
- 0
glance/db/sqlalchemy/alembic_migrations/alembic.ini View File

@@ -0,0 +1,69 @@
1
+# A generic, single database configuration.
2
+
3
+[alembic]
4
+# path to migration scripts
5
+script_location = %(here)s
6
+
7
+# template used to generate migration files
8
+# file_template = %%(rev)s_%%(slug)s
9
+
10
+# max length of characters to apply to the
11
+# "slug" field
12
+#truncate_slug_length = 40
13
+
14
+# set to 'true' to run the environment during
15
+# the 'revision' command, regardless of autogenerate
16
+# revision_environment = false
17
+
18
+# set to 'true' to allow .pyc and .pyo files without
19
+# a source .py file to be detected as revisions in the
20
+# versions/ directory
21
+# sourceless = false
22
+
23
+# version location specification; this defaults
24
+# to alembic_migrations/versions.  When using multiple version
25
+# directories, initial revisions must be specified with --version-path
26
+# version_locations = %(here)s/bar %(here)s/bat alembic_migrations/versions
27
+
28
+# the output encoding used when revision files
29
+# are written from script.py.mako
30
+# output_encoding = utf-8
31
+
32
+# Uncomment and update to your sql connection string if wishing to run
33
+# alembic directly from command line
34
+#sqlalchemy.url =
35
+
36
+# Logging configuration
37
+[loggers]
38
+keys = root,sqlalchemy,alembic
39
+
40
+[handlers]
41
+keys = console
42
+
43
+[formatters]
44
+keys = generic
45
+
46
+[logger_root]
47
+level = WARN
48
+handlers = console
49
+qualname =
50
+
51
+[logger_sqlalchemy]
52
+level = WARN
53
+handlers =
54
+qualname = sqlalchemy.engine
55
+
56
+[logger_alembic]
57
+level = INFO
58
+handlers =
59
+qualname = alembic
60
+
61
+[handler_console]
62
+class = StreamHandler
63
+args = (sys.stderr,)
64
+level = NOTSET
65
+formatter = generic
66
+
67
+[formatter_generic]
68
+format = %(levelname)-5.5s [%(name)s] %(message)s
69
+datefmt = %H:%M:%S

+ 92
- 0
glance/db/sqlalchemy/alembic_migrations/env.py View File

@@ -0,0 +1,92 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2013 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+from __future__ import with_statement
17
+from logging import config as log_config
18
+
19
+from alembic import context
20
+from sqlalchemy import engine_from_config, pool
21
+
22
+from glance.db.sqlalchemy import models
23
+from glance.db.sqlalchemy import models_glare
24
+from glance.db.sqlalchemy import models_metadef
25
+
26
+# this is the Alembic Config object, which provides
27
+# access to the values within the .ini file in use.
28
+config = context.config
29
+
30
+# other values from the config, defined by the needs of env.py,
31
+# can be acquired:
32
+# my_important_option = config.get_main_option("my_important_option")
33
+# ... etc.
34
+
35
+# Interpret the config file for Python logging.
36
+# This line sets up loggers basically.
37
+log_config.fileConfig(config.config_file_name)
38
+
39
+# add your model's MetaData object here
40
+# for 'autogenerate' support
41
+target_metadata = models.BASE.metadata
42
+for table in models_glare.BASE.metadata.sorted_tables:
43
+    target_metadata._add_table(table.name, table.schema, table)
44
+for table in models_metadef.BASE_DICT.metadata.sorted_tables:
45
+    target_metadata._add_table(table.name, table.schema, table)
46
+
47
+
48
+def run_migrations_offline():
49
+    """Run migrations in 'offline' mode.
50
+
51
+    This configures the context with just a URL
52
+    and not an Engine, though an Engine is acceptable
53
+    here as well.  By skipping the Engine creation
54
+    we don't even need a DBAPI to be available.
55
+
56
+    Calls to context.execute() here emit the given string to the
57
+    script output.
58
+
59
+    """
60
+    url = config.get_main_option("sqlalchemy.url")
61
+    context.configure(
62
+        url=url, target_metadata=target_metadata, literal_binds=True)
63
+
64
+    with context.begin_transaction():
65
+        context.run_migrations()
66
+
67
+
68
+def run_migrations_online():
69
+    """Run migrations in 'online' mode.
70
+
71
+    In this scenario we need to create an Engine
72
+    and associate a connection with the context.
73
+
74
+    """
75
+    connectable = engine_from_config(
76
+        config.get_section(config.config_ini_section),
77
+        prefix='sqlalchemy.',
78
+        poolclass=pool.NullPool)
79
+
80
+    with connectable.connect() as connection:
81
+        context.configure(
82
+            connection=connection,
83
+            target_metadata=target_metadata
84
+        )
85
+
86
+        with context.begin_transaction():
87
+            context.run_migrations()
88
+
89
+if context.is_offline_mode():
90
+    run_migrations_offline()
91
+else:
92
+    run_migrations_online()

+ 20
- 0
glance/db/sqlalchemy/alembic_migrations/migrate.cfg View File

@@ -0,0 +1,20 @@
1
+[db_settings]
2
+# Used to identify which repository this database is versioned under.
3
+# You can use the name of your project.
4
+repository_id=Glance Migrations
5
+
6
+# The name of the database table used to track the schema version.
7
+# This name shouldn't already be used by your project.
8
+# If this is changed once a database is under version control, you'll need to
9
+# change the table name in each database too.
10
+version_table=alembic_version
11
+
12
+# When committing a change script, Migrate will attempt to generate the
13
+# sql for all supported databases; normally, if one of them fails - probably
14
+# because you don't have that database installed - it is ignored and the
15
+# commit continues, perhaps ending successfully.
16
+# Databases in this list MUST compile successfully during a commit, or the
17
+# entire commit will fail. List the databases your application will actually
18
+# be using to ensure your updates to that database work properly.
19
+# This must be a list; example: ['postgres','sqlite']
20
+required_dbs=[]

+ 20
- 0
glance/db/sqlalchemy/alembic_migrations/script.py.mako View File

@@ -0,0 +1,20 @@
1
+"""${message}
2
+
3
+Revision ID: ${up_revision}
4
+Revises: ${down_revision | comma,n}
5
+Create Date: ${create_date}
6
+
7
+"""
8
+
9
+# revision identifiers, used by Alembic.
10
+revision = ${repr(up_revision)}
11
+down_revision = ${repr(down_revision)}
12
+branch_labels = ${repr(branch_labels)}
13
+depends_on = ${repr(depends_on)}
14
+
15
+from alembic import op
16
+import sqlalchemy as sa
17
+${imports if imports else ""}
18
+
19
+def upgrade():
20
+    ${upgrades if upgrades else "pass"}

+ 40
- 0
glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py View File

@@ -0,0 +1,40 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2013 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+"""liberty initial
17
+
18
+Revision ID: liberty
19
+Revises:
20
+Create Date: 2016-08-03 16:06:59.657433
21
+
22
+"""
23
+
24
+from glance.db.sqlalchemy.alembic_migrations import add_artifacts_tables
25
+from glance.db.sqlalchemy.alembic_migrations import add_images_tables
26
+from glance.db.sqlalchemy.alembic_migrations import add_metadefs_tables
27
+from glance.db.sqlalchemy.alembic_migrations import add_tasks_tables
28
+
29
+# revision identifiers, used by Alembic.
30
+revision = 'liberty'
31
+down_revision = None
32
+branch_labels = None
33
+depends_on = None
34
+
35
+
36
+def upgrade():
37
+    add_images_tables.upgrade()
38
+    add_tasks_tables.upgrade()
39
+    add_metadefs_tables.upgrade()
40
+    add_artifacts_tables.upgrade()

+ 47
- 0
glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py View File

@@ -0,0 +1,47 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2013 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+"""add index on created_at and updated_at columns of 'images' table
17
+
18
+Revision ID: mitaka01
19
+Revises: liberty
20
+Create Date: 2016-08-03 17:19:35.306161
21
+
22
+"""
23
+
24
+from alembic import op
25
+from sqlalchemy import MetaData, Table, Index
26
+
27
+
28
+# revision identifiers, used by Alembic.
29
+revision = 'mitaka01'
30
+down_revision = 'liberty'
31
+branch_labels = None
32
+depends_on = None
33
+
34
+CREATED_AT_INDEX = 'created_at_image_idx'
35
+UPDATED_AT_INDEX = 'updated_at_image_idx'
36
+
37
+
38
+def upgrade():
39
+    migrate_engine = op.get_bind()
40
+    meta = MetaData(bind=migrate_engine)
41
+
42
+    images = Table('images', meta, autoload=True)
43
+
44
+    created_index = Index(CREATED_AT_INDEX, images.c.created_at)
45
+    created_index.create(migrate_engine)
46
+    updated_index = Index(UPDATED_AT_INDEX, images.c.updated_at)
47
+    updated_index.create(migrate_engine)

+ 42
- 0
glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py View File

@@ -0,0 +1,42 @@
1
+# Copyright 2016 Rackspace
2
+# Copyright 2013 Intel Corporation
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+"""update metadef os_nova_server
17
+
18
+Revision ID: mitaka02
19
+Revises: mitaka01
20
+Create Date: 2016-08-03 17:23:23.041663
21
+
22
+"""
23
+
24
+from alembic import op
25
+from sqlalchemy import MetaData, Table
26
+
27
+
28
+# revision identifiers, used by Alembic.
29
+revision = 'mitaka02'
30
+down_revision = 'mitaka01'
31
+branch_labels = None
32
+depends_on = None
33
+
34
+
35
+def upgrade():
36
+    migrate_engine = op.get_bind()
37
+    meta = MetaData(bind=migrate_engine)
38
+
39
+    resource_types_table = Table('metadef_resource_types', meta, autoload=True)
40
+
41
+    resource_types_table.update(values={'name': 'OS::Nova::Server'}).where(
42
+        resource_types_table.c.name == 'OS::Nova::Instance').execute()

+ 72
- 0
glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py View File

@@ -0,0 +1,72 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+"""add visibility to and remove is_public from images
14
+
15
+Revision ID: ocata01
16
+Revises: mitaka02
17
+Create Date: 2017-01-20 12:58:16.647499
18
+
19
+"""
20
+
21
+import os
22
+
23
+from alembic import op
24
+from sqlalchemy import Column, Enum, MetaData, select, Table, not_, and_
25
+import sqlparse
26
+
27
+# revision identifiers, used by Alembic.
28
+revision = 'ocata01'
29
+down_revision = 'mitaka02'
30
+branch_labels = None
31
+depends_on = None
32
+
33
+
34
+def upgrade():
35
+    migrate_engine = op.get_bind()
36
+    meta = MetaData(bind=migrate_engine)
37
+
38
+    engine_name = migrate_engine.engine.name
39
+    if engine_name == 'sqlite':
40
+        sql_file = os.path.splitext(__file__)[0]
41
+        sql_file += '.sql'
42
+        with open(sql_file, 'r') as sqlite_script:
43
+            sql = sqlparse.format(sqlite_script.read(), strip_comments=True)
44
+            for statement in sqlparse.split(sql):
45
+                op.execute(statement)
46
+        return
47
+
48
+    enum = Enum('private', 'public', 'shared', 'community', metadata=meta,
49
+                name='image_visibility')
50
+    enum.create()
51
+    v_col = Column('visibility', enum, nullable=False, server_default='shared')
52
+    op.add_column('images', v_col)
53
+
54
+    op.create_index('visibility_image_idx', 'images', ['visibility'])
55
+
56
+    images = Table('images', meta, autoload=True)
57
+    images.update(values={'visibility': 'public'}).where(
58
+        images.c.is_public).execute()
59
+
60
+    image_members = Table('image_members', meta, autoload=True)
61
+
62
+    # NOTE(dharinic): Mark all the non-public images as 'private' first
63
+    images.update().values(visibility='private').where(
64
+        not_(images.c.is_public)).execute()
65
+    # NOTE(dharinic): Identify 'shared' images from the above
66
+    images.update().values(visibility='shared').where(and_(
67
+        images.c.visibility == 'private', images.c.id.in_(select(
68
+            [image_members.c.image_id]).distinct().where(
69
+                not_(image_members.c.deleted))))).execute()
70
+
71
+    op.drop_index('ix_images_is_public', 'images')
72
+    op.drop_column('images', 'is_public')

+ 162
- 0
glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql View File

@@ -0,0 +1,162 @@
1
+CREATE TEMPORARY TABLE images_backup (
2
+    id VARCHAR(36) NOT NULL,
3
+    name VARCHAR(255),
4
+    size INTEGER,
5
+    status VARCHAR(30) NOT NULL,
6
+    is_public BOOLEAN NOT NULL,
7
+    created_at DATETIME NOT NULL,
8
+    updated_at DATETIME,
9
+    deleted_at DATETIME,
10
+    deleted BOOLEAN NOT NULL,
11
+    disk_format VARCHAR(20),
12
+    container_format VARCHAR(20),
13
+    checksum VARCHAR(32),
14
+    owner VARCHAR(255),
15
+    min_disk INTEGER NOT NULL,
16
+    min_ram INTEGER NOT NULL,
17
+    protected BOOLEAN DEFAULT 0 NOT NULL,
18
+    virtual_size INTEGER,
19
+    PRIMARY KEY (id),
20
+    CHECK (is_public IN (0, 1)),
21
+    CHECK (deleted IN (0, 1)),
22
+    CHECK (protected IN (0, 1))
23
+);
24
+
25
+INSERT INTO images_backup
26
+    SELECT id,
27
+        name,
28
+        size,
29
+        status,
30
+        is_public,
31
+        created_at,
32
+        updated_at,
33
+        deleted_at,
34
+        deleted,
35
+        disk_format,
36
+        container_format,
37
+        checksum,
38
+        owner,
39
+        min_disk,
40
+        min_ram,
41
+        protected,
42
+        virtual_size
43
+    FROM images;
44
+
45
+DROP TABLE images;
46
+
47
+CREATE TABLE images (
48
+    id VARCHAR(36) NOT NULL,
49
+    name VARCHAR(255),
50
+    size INTEGER,
51
+    status VARCHAR(30) NOT NULL,
52
+    created_at DATETIME NOT NULL,
53
+    updated_at DATETIME,
54
+    deleted_at DATETIME,
55
+    deleted BOOLEAN NOT NULL,
56
+    disk_format VARCHAR(20),
57
+    container_format VARCHAR(20),
58
+    checksum VARCHAR(32),
59
+    owner VARCHAR(255),
60
+    min_disk INTEGER NOT NULL,
61
+    min_ram INTEGER NOT NULL,
62
+    protected BOOLEAN DEFAULT 0 NOT NULL,
63
+    virtual_size INTEGER,
64
+    visibility VARCHAR(9) DEFAULT 'shared' NOT NULL,
65
+    PRIMARY KEY (id),
66
+    CHECK (deleted IN (0, 1)),
67
+    CHECK (protected IN (0, 1)),
68
+    CONSTRAINT image_visibility CHECK (visibility IN ('private', 'public', 'shared', 'community'))
69
+);
70
+
71
+CREATE INDEX checksum_image_idx ON images (checksum);
72
+CREATE INDEX visibility_image_idx ON images (visibility);
73
+CREATE INDEX ix_images_deleted ON images (deleted);
74
+CREATE INDEX owner_image_idx ON images (owner);
75
+CREATE INDEX created_at_image_idx ON images (created_at);
76
+CREATE INDEX updated_at_image_idx ON images (updated_at);
77
+
78
+-- Copy over all the 'public' rows
79
+
80
+INSERT INTO images (
81
+    id,
82
+    name,
83
+    size,
84
+    status,
85
+    created_at,
86
+    updated_at,
87
+    deleted_at,
88
+    deleted,
89
+    disk_format,
90
+    container_format,
91
+    checksum,
92
+    owner,
93
+    min_disk,
94
+    min_ram,
95
+    protected,
96
+    virtual_size
97
+    )
98
+    SELECT id,
99
+        name,
100
+        size,
101
+        status,
102
+        created_at,
103
+        updated_at,
104
+        deleted_at,
105
+        deleted,
106
+        disk_format,
107
+        container_format,
108
+        checksum,
109
+        owner,
110
+        min_disk,
111
+        min_ram,
112
+        protected,
113
+        virtual_size
114
+    FROM images_backup
115
+    WHERE is_public=1;
116
+
117
+
118
+UPDATE images SET visibility='public';
119
+
120
+-- Now copy over the 'private' rows
121
+
122
+INSERT INTO images (
123
+    id,
124
+    name,
125
+    size,
126
+    status,
127
+    created_at,
128
+    updated_at,
129
+    deleted_at,
130
+    deleted,
131
+    disk_format,
132
+    container_format,
133
+    checksum,
134
+    owner,
135
+    min_disk,
136
+    min_ram,
137
+    protected,
138
+    virtual_size
139
+    )
140
+    SELECT id,
141
+        name,
142
+        size,
143
+        status,
144
+        created_at,
145
+        updated_at,
146
+        deleted_at,
147
+        deleted,
148
+        disk_format,
149
+        container_format,
150
+        checksum,
151
+        owner,
152
+        min_disk,
153
+        min_ram,
154
+        protected,
155
+        virtual_size
156
+    FROM images_backup
157
+    WHERE is_public=0;
158
+
159
+UPDATE images SET visibility='private' WHERE visibility='shared';
160
+UPDATE images SET visibility='shared' WHERE visibility='private' AND id IN (SELECT DISTINCT image_id FROM image_members WHERE deleted != 1);
161
+
162
+DROP TABLE images_backup;

+ 42
- 70
glance/tests/unit/test_manage.py View File

@@ -15,11 +15,8 @@
15 15
 
16 16
 import fixtures
17 17
 import mock
18
-from oslo_db.sqlalchemy import migration
19
-from six.moves import StringIO
20 18
 
21 19
 from glance.cmd import manage
22
-from glance.db import migration as db_migration
23 20
 from glance.db.sqlalchemy import api as db_api
24 21
 from glance.db.sqlalchemy import metadata as db_metadata
25 22
 from glance.tests import utils as test_utils
@@ -51,48 +48,35 @@ class TestManageBase(test_utils.BaseTestCase):
51 48
 
52 49
 class TestLegacyManage(TestManageBase):
53 50
 
54
-    @mock.patch.object(migration, 'db_version')
55
-    def test_legacy_db_version(self, db_version):
56
-        with mock.patch('sys.stdout', new_callable=StringIO):
57
-            self._main_test_helper(['glance.cmd.manage', 'db_version'],
58
-                                   migration.db_version,
59
-                                   db_api.get_engine(),
60
-                                   db_migration.MIGRATE_REPO_PATH, 0)
51
+    @mock.patch.object(manage.DbCommands, 'version')
52
+    def test_legacy_db_version(self, db_upgrade):
53
+        self._main_test_helper(['glance.cmd.manage', 'db_version'],
54
+                               manage.DbCommands.version)
61 55
 
62
-    @mock.patch.object(migration, 'db_sync')
56
+    @mock.patch.object(manage.DbCommands, 'sync')
63 57
     def test_legacy_db_sync(self, db_sync):
64 58
         self._main_test_helper(['glance.cmd.manage', 'db_sync'],
65
-                               migration.db_sync,
66
-                               db_api.get_engine(),
67
-                               db_migration.MIGRATE_REPO_PATH, None)
59
+                               manage.DbCommands.sync, None)
68 60
 
69
-    @mock.patch.object(migration, 'db_sync')
70
-    def test_legacy_db_upgrade(self, db_sync):
61
+    @mock.patch.object(manage.DbCommands, 'upgrade')
62
+    def test_legacy_db_upgrade(self, db_upgrade):
71 63
         self._main_test_helper(['glance.cmd.manage', 'db_upgrade'],
72
-                               migration.db_sync,
73
-                               db_api.get_engine(),
74
-                               db_migration.MIGRATE_REPO_PATH, None)
64
+                               manage.DbCommands.upgrade, None)
75 65
 
76
-    @mock.patch.object(migration, 'db_version_control')
66
+    @mock.patch.object(manage.DbCommands, 'version_control')
77 67
     def test_legacy_db_version_control(self, db_version_control):
78 68
         self._main_test_helper(['glance.cmd.manage', 'db_version_control'],
79
-                               migration.db_version_control,
80
-                               db_api.get_engine(),
81
-                               db_migration.MIGRATE_REPO_PATH, None)
69
+                               manage.DbCommands.version_control, None)
82 70
 
83
-    @mock.patch.object(migration, 'db_sync')
71
+    @mock.patch.object(manage.DbCommands, 'sync')
84 72
     def test_legacy_db_sync_version(self, db_sync):
85
-        self._main_test_helper(['glance.cmd.manage', 'db_sync', '20'],
86
-                               migration.db_sync,
87
-                               db_api.get_engine(),
88
-                               db_migration.MIGRATE_REPO_PATH, '20')
73
+        self._main_test_helper(['glance.cmd.manage', 'db_sync', 'liberty'],
74
+                               manage.DbCommands.sync, 'liberty')
89 75
 
90
-    @mock.patch.object(migration, 'db_sync')
91
-    def test_legacy_db_upgrade_version(self, db_sync):
92
-        self._main_test_helper(['glance.cmd.manage', 'db_upgrade', '20'],
93
-                               migration.db_sync,
94
-                               db_api.get_engine(),
95
-                               db_migration.MIGRATE_REPO_PATH, '20')
76
+    @mock.patch.object(manage.DbCommands, 'upgrade')
77
+    def test_legacy_db_upgrade_version(self, db_upgrade):
78
+        self._main_test_helper(['glance.cmd.manage', 'db_upgrade', 'liberty'],
79
+                               manage.DbCommands.upgrade, 'liberty')
96 80
 
97 81
     def test_db_metadefs_unload(self):
98 82
         db_metadata.db_unload_metadefs = mock.Mock()
@@ -157,48 +141,36 @@ class TestLegacyManage(TestManageBase):
157 141
 
158 142
 class TestManage(TestManageBase):
159 143
 
160
-    @mock.patch.object(migration, 'db_version')
161
-    def test_db_version(self, db_version):
162
-        with mock.patch('sys.stdout', new_callable=StringIO):
163
-            self._main_test_helper(['glance.cmd.manage', 'db', 'version'],
164
-                                   migration.db_version,
165
-                                   db_api.get_engine(),
166
-                                   db_migration.MIGRATE_REPO_PATH, 0)
144
+    @mock.patch.object(manage.DbCommands, 'version')
145
+    def test_db_version(self, version):
146
+        self._main_test_helper(['glance.cmd.manage', 'db', 'version'],
147
+                               manage.DbCommands.version)
167 148
 
168
-    @mock.patch.object(migration, 'db_sync')
169
-    def test_db_sync(self, db_sync):
149
+    @mock.patch.object(manage.DbCommands, 'sync')
150
+    def test_db_sync(self, sync):
170 151
         self._main_test_helper(['glance.cmd.manage', 'db', 'sync'],
171
-                               migration.db_sync,
172
-                               db_api.get_engine(),
173
-                               db_migration.MIGRATE_REPO_PATH, None)
152
+                               manage.DbCommands.sync)
174 153
 
175
-    @mock.patch.object(migration, 'db_sync')
176
-    def test_db_upgrade(self, db_sync):
154
+    @mock.patch.object(manage.DbCommands, 'upgrade')
155
+    def test_db_upgrade(self, upgrade):
177 156
         self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade'],
178
-                               migration.db_sync,
179
-                               db_api.get_engine(),
180
-                               db_migration.MIGRATE_REPO_PATH, None)
157
+                               manage.DbCommands.upgrade)
181 158
 
182
-    @mock.patch.object(migration, 'db_version_control')
183
-    def test_db_version_control(self, db_version_control):
159
+    @mock.patch.object(manage.DbCommands, 'version_control')
160
+    def test_db_version_control(self, version_control):
184 161
         self._main_test_helper(['glance.cmd.manage', 'db', 'version_control'],
185
-                               migration.db_version_control,
186
-                               db_api.get_engine(),
187
-                               db_migration.MIGRATE_REPO_PATH, None)
188
-
189
-    @mock.patch.object(migration, 'db_sync')
190
-    def test_db_sync_version(self, db_sync):
191
-        self._main_test_helper(['glance.cmd.manage', 'db', 'sync', '20'],
192
-                               migration.db_sync,
193
-                               db_api.get_engine(),
194
-                               db_migration.MIGRATE_REPO_PATH, '20')
195
-
196
-    @mock.patch.object(migration, 'db_sync')
197
-    def test_db_upgrade_version(self, db_sync):
198
-        self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade', '20'],
199
-                               migration.db_sync,
200
-                               db_api.get_engine(),
201
-                               db_migration.MIGRATE_REPO_PATH, '20')
162
+                               manage.DbCommands.version_control)
163
+
164
+    @mock.patch.object(manage.DbCommands, 'sync')
165
+    def test_db_sync_version(self, sync):
166
+        self._main_test_helper(['glance.cmd.manage', 'db', 'sync', 'liberty'],
167
+                               manage.DbCommands.sync, 'liberty')
168
+
169
+    @mock.patch.object(manage.DbCommands, 'upgrade')
170
+    def test_db_upgrade_version(self, upgrade):
171
+        self._main_test_helper(['glance.cmd.manage', 'db',
172
+                                'upgrade', 'liberty'],
173
+                               manage.DbCommands.upgrade, 'liberty')
202 174
 
203 175
     def test_db_metadefs_unload(self):
204 176
         db_metadata.db_unload_metadefs = mock.Mock()

+ 2
- 0
requirements.txt View File

@@ -12,6 +12,8 @@ Routes!=2.0,!=2.1,!=2.3.0,>=1.12.3;python_version=='2.7' # MIT
12 12
 Routes!=2.0,!=2.3.0,>=1.12.3;python_version!='2.7' # MIT
13 13
 WebOb>=1.6.0 # MIT
14 14
 sqlalchemy-migrate>=0.9.6 # Apache-2.0
15
+sqlparse>=0.2.2 # BSD
16
+alembic>=0.8.10 # MIT
15 17
 httplib2>=0.7.5 # MIT
16 18
 pycrypto>=2.6 # Public Domain
17 19
 oslo.config!=3.18.0,>=3.14.0 # Apache-2.0

Loading…
Cancel
Save