Port Glance Migrations to Alembic

This change proposes the use of Alembic to manage Glance migrations.
* Introduce new directory ``alembic_migrations`` under
  ``glance/db/sqlalchemy``. This directory is the home for all glance
  migrations henceforth. All the migration scripts reside under
  ``versions`` directory.
* All the migrations up to Liberty are consolidated into one migration
  called ``liberty_initial`` as those migrations are not supported
  any more. Mitaka migrations are retained but under a different naming
  convention.
* All the glance manage db commands are changed appropriately. They now
  use alembic to perform operations such as ``version``, ``upgrade``,
  ``sync`` and ``version_control``.
* The database versions are not numerical any more. They are the revision
  ID of the last migration applied on the database. Since we don't
  support migrations before Mitaka, the Liberty version ``42`` will now
  appear as ``liberty``. Migration ``43`` and ``44`` in Mitaka appear as
  ``mitaka01`` and ``mitaka02`` respectively.
* When one performs a ``sync`` or ``upgrade`` command, the database is
  first stamped with an equivalent alembic version before upgrading.
* The older migration scripts are retained so that users can correlate
  with the new migrations. Also, it is probably safe to retain them until
  the alembic migrations become stable. Similarly, the ``migrate_version``
  table is not removed yet.

Partially-Implements: blueprint alembic-migrations

Change-Id: Ie8594ff339a13bf190aefa308f54e97ee20ecfa2
Co-Authored-By: Alexander Bashmakov <alexander.bashmakov@intel.com>
Depends-On: I1596499529af249bc48dfe859bbd31e90c48a5e0

doc/source/db.rst

@@ -29,9 +29,10 @@ The commands should be executed as a subcommand of 'db':
 Sync the Database
 -----------------
 
-    glance-manage db sync <version> <current_version>
+    glance-manage db sync <VERSION>
 
-Place a database under migration control and upgrade, creating it first if necessary.
+Place an existing database under migration control and upgrade it to the
+specified VERSION.
 
 
 Determining the Database Version

doc/source/man/glancemanage.rst

@@ -53,9 +53,9 @@ COMMANDS
   **db_version_control**
         Place the database under migration control.
 
-  **db_sync <VERSION> <CURRENT_VERSION>**
-        Place a database under migration control and upgrade, creating
-        it first if necessary.
+  **db_sync <VERSION>**
+        Place an existing database under migration control and upgrade it to
+        the specified VERSION.
 
   **db_export_metadefs [PATH | PREFIX]**
         Export the metadata definitions into json format. By default the
@@ -80,10 +80,6 @@ OPTIONS
 
   .. include:: general_options.rst
 
-  **--sql_connection=CONN_STRING**
-        A proper SQLAlchemy connection string as described
-        `here <http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html?highlight=engine#sqlalchemy.create_engine>`_
-
 .. include:: footer.rst
 
 CONFIGURATION

glance/cmd/manage.py

@@ -39,8 +39,9 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
 if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
     sys.path.insert(0, possible_topdir)
 
+from alembic import command as alembic_command
+
 from oslo_config import cfg
-from oslo_db.sqlalchemy import migration
 from oslo_log import log as logging
 from oslo_utils import encodeutils
 import six
@@ -49,6 +50,7 @@ from glance.common import config
 from glance.common import exception
 from glance import context
 from glance.db import migration as db_migration
+from glance.db.sqlalchemy import alembic_migrations
 from glance.db.sqlalchemy import api as db_api
 from glance.db.sqlalchemy import metadata
 from glance.i18n import _
@@ -73,39 +75,56 @@ class DbCommands(object):
 
     def version(self):
         """Print database's current migration level"""
-        print(migration.db_version(db_api.get_engine(),
-                                   db_migration.MIGRATE_REPO_PATH,
-                                   db_migration.INIT_VERSION))
+        current_heads = alembic_migrations.get_current_alembic_heads()
+        if current_heads:
+            # Migrations are managed by alembic
+            for head in current_heads:
+                print(head)
+        else:
+            # Migrations are managed by legacy versioning scheme
+            print(_('Database is either not under migration control or under '
+                    'legacy migration control, please run '
+                    '"glance-manage db sync" to place the database under '
+                    'alembic migration control.'))
 
     @args('--version', metavar='<version>', help='Database version')
-    def upgrade(self, version=None):
+    def upgrade(self, version='heads'):
         """Upgrade the database's migration level"""
-        migration.db_sync(db_api.get_engine(),
-                          db_migration.MIGRATE_REPO_PATH,
-                          version)
+        self.sync(version)
 
     @args('--version', metavar='<version>', help='Database version')
-    def version_control(self, version=None):
+    def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION):
         """Place a database under migration control"""
-        migration.db_version_control(db_api.get_engine(),
-                                     db_migration.MIGRATE_REPO_PATH,
-                                     version)
+
+        if version is None:
+            version = db_migration.ALEMBIC_INIT_VERSION
+
+        a_config = alembic_migrations.get_alembic_config()
+        alembic_command.stamp(a_config, version)
+        print(_("Placed database under migration control at "
+                "revision:"), version)
 
     @args('--version', metavar='<version>', help='Database version')
-    @args('--current_version', metavar='<version>',
-          help='Current Database version')
-    def sync(self, version=None, current_version=None):
+    def sync(self, version='heads'):
         """
-        Place a database under migration control and upgrade it,
-        creating first if necessary.
+        Place an existing database under migration control and upgrade it.
         """
-        if current_version not in (None, 'None'):
-            migration.db_version_control(db_api.get_engine(),
-                                         db_migration.MIGRATE_REPO_PATH,
-                                         version=current_version)
-        migration.db_sync(db_api.get_engine(),
-                          db_migration.MIGRATE_REPO_PATH,
-                          version)
+        if version is None:
+            version = 'heads'
+
+        alembic_migrations.place_database_under_alembic_control()
+
+        a_config = alembic_migrations.get_alembic_config()
+        alembic_command.upgrade(a_config, version)
+        heads = alembic_migrations.get_current_alembic_heads()
+        if heads is None:
+            raise Exception("Database sync failed")
+        revs = ", ".join(heads)
+        if version is 'heads':
+            print(_("Upgraded database, current revision(s):"), revs)
+        else:
+            print(_('Upgraded database to: %(v)s, current revision(s): %(r)s')
+                  % {'v': version, 'r': revs})
 
     @args('--path', metavar='<path>', help='Path to the directory or file '
                                            'where json metadata is stored')
@@ -179,15 +198,14 @@ class DbLegacyCommands(object):
     def version(self):
         self.command_object.version()
 
-    def upgrade(self, version=None):
+    def upgrade(self, version='heads'):
         self.command_object.upgrade(CONF.command.version)
 
-    def version_control(self, version=None):
+    def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION):
         self.command_object.version_control(CONF.command.version)
 
-    def sync(self, version=None, current_version=None):
-        self.command_object.sync(CONF.command.version,
-                                 CONF.command.current_version)
+    def sync(self, version='heads'):
+        self.command_object.sync(CONF.command.version)
 
     def load_metadefs(self, path=None, merge=False,
                       prefer_new=False, overwrite=False):
@@ -224,7 +242,6 @@ def add_legacy_command_parsers(command_object, subparsers):
     parser = subparsers.add_parser('db_sync')
     parser.set_defaults(action_fn=legacy_command_object.sync)
     parser.add_argument('version', nargs='?')
-    parser.add_argument('current_version', nargs='?')
     parser.set_defaults(action='db_sync')
 
     parser = subparsers.add_parser('db_load_metadefs')

glance/db/migration.py

@@ -45,6 +45,7 @@ def get_backend():
                     cfg.CONF.database.backend).driver
     return _IMPL
 
+ALEMBIC_INIT_VERSION = 'liberty'
 INIT_VERSION = 0
 
 MIGRATE_REPO_PATH = os.path.join(

glance/db/sqlalchemy/alembic_migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.

glance/db/sqlalchemy/alembic_migrations/__init__.py

@@ -0,0 +1,99 @@
+# Copyright 2016 Rackspace
+# Copyright 2013 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import sys
+
+from alembic import command as alembic_command
+from alembic import config as alembic_config
+from alembic import migration as alembic_migration
+from oslo_db import exception as db_exception
+from oslo_db.sqlalchemy import migration
+
+from glance.db import migration as db_migration
+from glance.db.sqlalchemy import api as db_api
+from glance.i18n import _
+
+
+def get_alembic_config():
+    """Return a valid alembic config object"""
+    ini_path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
+    config = alembic_config.Config(os.path.abspath(ini_path))
+    dbconn = str(db_api.get_engine().url)
+    config.set_main_option('sqlalchemy.url', dbconn)
+    return config
+
+
+def get_current_alembic_heads():
+    """Return current heads (if any) from the alembic migration table"""
+    engine = db_api.get_engine()
+    with engine.connect() as conn:
+        context = alembic_migration.MigrationContext.configure(conn)
+        heads = context.get_current_heads()
+        return heads
+
+
+def get_current_legacy_head():
+    try:
+        legacy_head = migration.db_version(db_api.get_engine(),
+                                           db_migration.MIGRATE_REPO_PATH,
+                                           db_migration.INIT_VERSION)
+    except db_exception.DbMigrationError:
+        legacy_head = None
+    return legacy_head
+
+
+def is_database_under_alembic_control():
+    if get_current_alembic_heads():
+        return True
+    return False
+
+
+def is_database_under_migrate_control():
+    if get_current_legacy_head():
+        return True
+    return False
+
+
+def place_database_under_alembic_control():
+    a_config = get_alembic_config()
+
+    if not is_database_under_migrate_control():
+        return
+
+    if not is_database_under_alembic_control():
+        print(_("Database is currently not under Alembic's migration "
+                "control."))
+        head = get_current_legacy_head()
+        if head == 42:
+            alembic_version = 'liberty'
+        elif head == 43:
+            alembic_version = 'mitaka01'
+        elif head == 44:
+            alembic_version = 'mitaka02'
+        elif head == 45:
+            alembic_version = 'ocata01'
+        elif head in range(1, 42):
+            print("Legacy head: ", head)
+            sys.exit(_("The current database version is not supported any "
+                       "more. Please upgrade to Liberty release first."))
+        else:
+            sys.exit(_("Unable to place database under Alembic's migration "
+                       "control. Unknown database state, can't proceed "
+                       "further."))
+
+        print(_("Placing database under Alembic's migration control at "
+                "revision:"), alembic_version)
+        alembic_command.stamp(a_config, alembic_version)

glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py

@@ -0,0 +1,224 @@
+# Copyright 2016 Rackspace
+# Copyright 2013 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from alembic import op
+from sqlalchemy.schema import (
+    Column, PrimaryKeyConstraint, ForeignKeyConstraint)
+
+from glance.db.sqlalchemy.migrate_repo.schema import (
+    Boolean, DateTime, Integer, BigInteger, String, Text, Numeric)  # noqa
+
+
+def _add_artifacts_table():
+    op.create_table('artifacts',
+                    Column('id', String(length=36), nullable=False),
+                    Column('name', String(length=255), nullable=False),
+                    Column('type_name', String(length=255), nullable=False),
+                    Column('type_version_prefix',
+                           BigInteger(),
+                           nullable=False),
+                    Column('type_version_suffix',
+                           String(length=255),
+                           nullable=True),
+                    Column('type_version_meta',
+                           String(length=255),
+                           nullable=True),
+                    Column('version_prefix', BigInteger(), nullable=False),
+                    Column('version_suffix',
+                           String(length=255),
+                           nullable=True),
+                    Column('version_meta', String(length=255), nullable=True),
+                    Column('description', Text(), nullable=True),
+                    Column('visibility', String(length=32), nullable=False),
+                    Column('state', String(length=32), nullable=False),
+                    Column('owner', String(length=255), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=False),
+                    Column('deleted_at', DateTime(), nullable=True),
+                    Column('published_at', DateTime(), nullable=True),
+                    PrimaryKeyConstraint('id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_artifact_name_and_version',
+                    'artifacts',
+                    ['name', 'version_prefix', 'version_suffix'],
+                    unique=False)
+    op.create_index('ix_artifact_owner', 'artifacts', ['owner'], unique=False)
+    op.create_index('ix_artifact_state', 'artifacts', ['state'], unique=False)
+    op.create_index('ix_artifact_type',
+                    'artifacts',
+                    ['type_name',
+                     'type_version_prefix',
+                     'type_version_suffix'],
+                    unique=False)
+    op.create_index('ix_artifact_visibility',
+                    'artifacts',
+                    ['visibility'],
+                    unique=False)
+
+
+def _add_artifact_blobs_table():
+    op.create_table('artifact_blobs',
+                    Column('id', String(length=36), nullable=False),
+                    Column('artifact_id', String(length=36), nullable=False),
+                    Column('size', BigInteger(), nullable=False),
+                    Column('checksum', String(length=32), nullable=True),
+                    Column('name', String(length=255), nullable=False),
+                    Column('item_key', String(length=329), nullable=True),
+                    Column('position', Integer(), nullable=True),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=False),
+                    ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_artifact_blobs_artifact_id',
+                    'artifact_blobs',
+                    ['artifact_id'],
+                    unique=False)
+    op.create_index('ix_artifact_blobs_name',
+                    'artifact_blobs',
+                    ['name'],
+                    unique=False)
+
+
+def _add_artifact_dependencies_table():
+    op.create_table('artifact_dependencies',
+                    Column('id', String(length=36), nullable=False),
+                    Column('artifact_source',
+                           String(length=36),
+                           nullable=False),
+                    Column('artifact_dest', String(length=36), nullable=False),
+                    Column('artifact_origin',
+                           String(length=36),
+                           nullable=False),
+                    Column('is_direct', Boolean(), nullable=False),
+                    Column('position', Integer(), nullable=True),
+                    Column('name', String(length=36), nullable=True),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=False),
+                    ForeignKeyConstraint(['artifact_dest'],
+                                         ['artifacts.id'], ),
+                    ForeignKeyConstraint(['artifact_origin'],
+                                         ['artifacts.id'], ),
+                    ForeignKeyConstraint(['artifact_source'],
+                                         ['artifacts.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_artifact_dependencies_dest_id',
+                    'artifact_dependencies',
+                    ['artifact_dest'],
+                    unique=False)
+    op.create_index('ix_artifact_dependencies_direct_dependencies',
+                    'artifact_dependencies',
+                    ['artifact_source', 'is_direct'],
+                    unique=False)
+    op.create_index('ix_artifact_dependencies_origin_id',
+                    'artifact_dependencies',
+                    ['artifact_origin'],
+                    unique=False)
+    op.create_index('ix_artifact_dependencies_source_id',
+                    'artifact_dependencies',
+                    ['artifact_source'],
+                    unique=False)
+
+
+def _add_artifact_properties_table():
+    op.create_table('artifact_properties',
+                    Column('id', String(length=36), nullable=False),
+                    Column('artifact_id', String(length=36), nullable=False),
+                    Column('name', String(length=255), nullable=False),
+                    Column('string_value', String(length=255), nullable=True),
+                    Column('int_value', Integer(), nullable=True),
+                    Column('numeric_value', Numeric(), nullable=True),
+                    Column('bool_value', Boolean(), nullable=True),
+                    Column('text_value', Text(), nullable=True),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=False),
+                    Column('position', Integer(), nullable=True),
+                    ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_artifact_properties_artifact_id',
+                    'artifact_properties',
+                    ['artifact_id'],
+                    unique=False)
+    op.create_index('ix_artifact_properties_name',
+                    'artifact_properties',
+                    ['name'],
+                    unique=False)
+
+
+def _add_artifact_tags_table():
+    op.create_table('artifact_tags',
+                    Column('id', String(length=36), nullable=False),
+                    Column('artifact_id', String(length=36), nullable=False),
+                    Column('value', String(length=255), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=False),
+                    ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_artifact_tags_artifact_id',
+                    'artifact_tags',
+                    ['artifact_id'],
+                    unique=False)
+    op.create_index('ix_artifact_tags_artifact_id_tag_value',
+                    'artifact_tags',
+                    ['artifact_id', 'value'],
+                    unique=False)
+
+
+def _add_artifact_blob_locations_table():
+    op.create_table('artifact_blob_locations',
+                    Column('id', String(length=36), nullable=False),
+                    Column('blob_id', String(length=36), nullable=False),
+                    Column('value', Text(), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=False),
+                    Column('position', Integer(), nullable=True),
+                    Column('status', String(length=36), nullable=True),
+                    ForeignKeyConstraint(['blob_id'], ['artifact_blobs.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_artifact_blob_locations_blob_id',
+                    'artifact_blob_locations',
+                    ['blob_id'],
+                    unique=False)
+
+
+def upgrade():
+    _add_artifacts_table()
+    _add_artifact_blobs_table()
+    _add_artifact_dependencies_table()
+    _add_artifact_properties_table()
+    _add_artifact_tags_table()
+    _add_artifact_blob_locations_table()

glance/db/sqlalchemy/alembic_migrations/add_images_tables.py

@@ -0,0 +1,201 @@
+# Copyright 2016 Rackspace
+# Copyright 2013 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from alembic import op
+from sqlalchemy import sql
+from sqlalchemy.schema import (
+    Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint)
+
+from glance.db.sqlalchemy.migrate_repo.schema import (
+    Boolean, DateTime, Integer, BigInteger, String, Text)  # noqa
+from glance.db.sqlalchemy.models import JSONEncodedDict
+
+
+def _add_images_table():
+    op.create_table('images',
+                    Column('id', String(length=36), nullable=False),
+                    Column('name', String(length=255), nullable=True),
+                    Column('size', BigInteger(), nullable=True),
+                    Column('status', String(length=30), nullable=False),
+                    Column('is_public', Boolean(), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    Column('deleted_at', DateTime(), nullable=True),
+                    Column('deleted', Boolean(), nullable=False),
+                    Column('disk_format', String(length=20), nullable=True),
+                    Column('container_format',
+                           String(length=20),
+                           nullable=True),
+                    Column('checksum', String(length=32), nullable=True),
+                    Column('owner', String(length=255), nullable=True),
+                    Column('min_disk', Integer(), nullable=False),
+                    Column('min_ram', Integer(), nullable=False),
+                    Column('protected',
+                           Boolean(),
+                           server_default=sql.false(),
+                           nullable=False),
+                    Column('virtual_size', BigInteger(), nullable=True),
+                    PrimaryKeyConstraint('id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('checksum_image_idx',
+                    'images',
+                    ['checksum'],
+                    unique=False)
+    op.create_index('ix_images_deleted',
+                    'images',
+                    ['deleted'],
+                    unique=False)
+    op.create_index('ix_images_is_public',
+                    'images',
+                    ['is_public'],
+                    unique=False)
+    op.create_index('owner_image_idx',
+                    'images',
+                    ['owner'],
+                    unique=False)
+
+
+def _add_image_properties_table():
+    op.create_table('image_properties',
+                    Column('id', Integer(), nullable=False),
+                    Column('image_id', String(length=36), nullable=False),
+                    Column('name', String(length=255), nullable=False),
+                    Column('value', Text(), nullable=True),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    Column('deleted_at', DateTime(), nullable=True),
+                    Column('deleted', Boolean(), nullable=False),
+                    PrimaryKeyConstraint('id'),
+                    ForeignKeyConstraint(['image_id'], ['images.id'], ),
+                    UniqueConstraint('image_id',
+                                     'name',
+                                     name='ix_image_properties_image_id_name'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_image_properties_deleted',
+                    'image_properties',
+                    ['deleted'],
+                    unique=False)
+    op.create_index('ix_image_properties_image_id',
+                    'image_properties',
+                    ['image_id'],
+                    unique=False)
+
+
+def _add_image_locations_table():
+    op.create_table('image_locations',
+                    Column('id', Integer(), nullable=False),
+                    Column('image_id', String(length=36), nullable=False),
+                    Column('value', Text(), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    Column('deleted_at', DateTime(), nullable=True),
+                    Column('deleted', Boolean(), nullable=False),
+                    Column('meta_data', JSONEncodedDict(), nullable=True),
+                    Column('status',
+                           String(length=30),
+                           server_default='active',
+                           nullable=False),
+                    PrimaryKeyConstraint('id'),
+                    ForeignKeyConstraint(['image_id'], ['images.id'], ),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_image_locations_deleted',
+                    'image_locations',
+                    ['deleted'],
+                    unique=False)
+    op.create_index('ix_image_locations_image_id',
+                    'image_locations',
+                    ['image_id'],
+                    unique=False)
+
+
+def _add_image_members_table():
+    deleted_member_constraint = 'image_members_image_id_member_deleted_at_key'
+    op.create_table('image_members',
+                    Column('id', Integer(), nullable=False),
+                    Column('image_id', String(length=36), nullable=False),
+                    Column('member', String(length=255), nullable=False),
+                    Column('can_share', Boolean(), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    Column('deleted_at', DateTime(), nullable=True),
+                    Column('deleted', Boolean(), nullable=False),
+                    Column('status',
+                           String(length=20),
+                           server_default='pending',
+                           nullable=False),
+                    ForeignKeyConstraint(['image_id'], ['images.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    UniqueConstraint('image_id',
+                                     'member',
+                                     'deleted_at',
+                                     name=deleted_member_constraint),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_image_members_deleted',
+                    'image_members',
+                    ['deleted'],
+                    unique=False)
+    op.create_index('ix_image_members_image_id',
+                    'image_members',
+                    ['image_id'],
+                    unique=False)
+    op.create_index('ix_image_members_image_id_member',
+                    'image_members',
+                    ['image_id', 'member'],
+                    unique=False)
+
+
+def _add_images_tags_table():
+    op.create_table('image_tags',
+                    Column('id', Integer(), nullable=False),
+                    Column('image_id', String(length=36), nullable=False),
+                    Column('value', String(length=255), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    Column('deleted_at', DateTime(), nullable=True),
+                    Column('deleted', Boolean(), nullable=False),
+                    ForeignKeyConstraint(['image_id'], ['images.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_image_tags_image_id',
+                    'image_tags',
+                    ['image_id'],
+                    unique=False)
+    op.create_index('ix_image_tags_image_id_tag_value',
+                    'image_tags',
+                    ['image_id', 'value'],
+                    unique=False)
+
+
+def upgrade():
+    _add_images_table()
+    _add_image_properties_table()
+    _add_image_locations_table()
+    _add_image_members_table()
+    _add_images_tags_table()

glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py

@@ -0,0 +1,171 @@
+# Copyright 2016 Rackspace
+# Copyright 2013 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from alembic import op
+from sqlalchemy.schema import (
+    Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint)
+
+from glance.db.sqlalchemy.migrate_repo.schema import (
+    Boolean, DateTime, Integer, String, Text)  # noqa
+from glance.db.sqlalchemy.models import JSONEncodedDict
+
+
+def _add_metadef_namespaces_table():
+    op.create_table('metadef_namespaces',
+                    Column('id', Integer(), nullable=False),
+                    Column('namespace', String(length=80), nullable=False),
+                    Column('display_name', String(length=80), nullable=True),
+                    Column('description', Text(), nullable=True),
+                    Column('visibility', String(length=32), nullable=True),
+                    Column('protected', Boolean(), nullable=True),
+                    Column('owner', String(length=255), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    PrimaryKeyConstraint('id'),
+                    UniqueConstraint('namespace',
+                                     name='uq_metadef_namespaces_namespace'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_metadef_namespaces_owner',
+                    'metadef_namespaces',
+                    ['owner'],
+                    unique=False)
+
+
+def _add_metadef_resource_types_table():
+    op.create_table('metadef_resource_types',
+                    Column('id', Integer(), nullable=False),
+                    Column('name', String(length=80), nullable=False),
+                    Column('protected', Boolean(), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    PrimaryKeyConstraint('id'),
+                    UniqueConstraint('name',
+                                     name='uq_metadef_resource_types_name'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+
+def _add_metadef_namespace_resource_types_table():
+    op.create_table('metadef_namespace_resource_types',
+                    Column('resource_type_id', Integer(), nullable=False),
+                    Column('namespace_id', Integer(), nullable=False),
+                    Column('properties_target',
+                           String(length=80),
+                           nullable=True),
+                    Column('prefix', String(length=80), nullable=True),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    ForeignKeyConstraint(['namespace_id'],
+                                         ['metadef_namespaces.id'], ),
+                    ForeignKeyConstraint(['resource_type_id'],
+                                         ['metadef_resource_types.id'], ),
+                    PrimaryKeyConstraint('resource_type_id', 'namespace_id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_metadef_ns_res_types_namespace_id',
+                    'metadef_namespace_resource_types',
+                    ['namespace_id'],
+                    unique=False)
+
+
+def _add_metadef_objects_table():
+    ns_id_name_constraint = 'uq_metadef_objects_namespace_id_name'
+
+    op.create_table('metadef_objects',
+                    Column('id', Integer(), nullable=False),
+                    Column('namespace_id', Integer(), nullable=False),
+                    Column('name', String(length=80), nullable=False),
+                    Column('description', Text(), nullable=True),
+                    Column('required', Text(), nullable=True),
+                    Column('json_schema', JSONEncodedDict(), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    ForeignKeyConstraint(['namespace_id'],
+                                         ['metadef_namespaces.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    UniqueConstraint('namespace_id',
+                                     'name',
+                                     name=ns_id_name_constraint),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_metadef_objects_name',
+                    'metadef_objects',
+                    ['name'],
+                    unique=False)
+
+
+def _add_metadef_properties_table():
+    ns_id_name_constraint = 'uq_metadef_properties_namespace_id_name'
+    op.create_table('metadef_properties',
+                    Column('id', Integer(), nullable=False),
+                    Column('namespace_id', Integer(), nullable=False),
+                    Column('name', String(length=80), nullable=False),
+                    Column('json_schema', JSONEncodedDict(), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    ForeignKeyConstraint(['namespace_id'],
+                                         ['metadef_namespaces.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    UniqueConstraint('namespace_id',
+                                     'name',
+                                     name=ns_id_name_constraint),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_metadef_properties_name',
+                    'metadef_properties',
+                    ['name'],
+                    unique=False)
+
+
+def _add_metadef_tags_table():
+    op.create_table('metadef_tags',
+                    Column('id', Integer(), nullable=False),
+                    Column('namespace_id', Integer(), nullable=False),
+                    Column('name', String(length=80), nullable=False),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    ForeignKeyConstraint(['namespace_id'],
+                                         ['metadef_namespaces.id'], ),
+                    PrimaryKeyConstraint('id'),
+                    UniqueConstraint('namespace_id',
+                                     'name',
+                                     name='uq_metadef_tags_namespace_id_name'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_metadef_tags_name',
+                    'metadef_tags',
+                    ['name'],
+                    unique=False)
+
+
+def upgrade():
+    _add_metadef_namespaces_table()
+    _add_metadef_resource_types_table()
+    _add_metadef_namespace_resource_types_table()
+    _add_metadef_objects_table()
+    _add_metadef_properties_table()
+    _add_metadef_tags_table()

glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py

@@ -0,0 +1,66 @@
+# Copyright 2016 Rackspace
+# Copyright 2013 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from alembic import op
+from sqlalchemy.schema import (
+    Column, PrimaryKeyConstraint, ForeignKeyConstraint)
+
+from glance.db.sqlalchemy.migrate_repo.schema import (
+    Boolean, DateTime, String, Text)  # noqa
+from glance.db.sqlalchemy.models import JSONEncodedDict
+
+
+def _add_tasks_table():
+    op.create_table('tasks',
+                    Column('id', String(length=36), nullable=False),
+                    Column('type', String(length=30), nullable=False),
+                    Column('status', String(length=30), nullable=False),
+                    Column('owner', String(length=255), nullable=False),
+                    Column('expires_at', DateTime(), nullable=True),
+                    Column('created_at', DateTime(), nullable=False),
+                    Column('updated_at', DateTime(), nullable=True),
+                    Column('deleted_at', DateTime(), nullable=True),
+                    Column('deleted', Boolean(), nullable=False),
+                    PrimaryKeyConstraint('id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+    op.create_index('ix_tasks_deleted', 'tasks', ['deleted'], unique=False)
+    op.create_index('ix_tasks_owner', 'tasks', ['owner'], unique=False)
+    op.create_index('ix_tasks_status', 'tasks', ['status'], unique=False)
+    op.create_index('ix_tasks_type', 'tasks', ['type'], unique=False)
+    op.create_index('ix_tasks_updated_at',
+                    'tasks',
+                    ['updated_at'],
+                    unique=False)
+
+
+def _add_task_info_table():
+    op.create_table('task_info',
+                    Column('task_id', String(length=36), nullable=False),
+                    Column('input', JSONEncodedDict(), nullable=True),
+                    Column('result', JSONEncodedDict(), nullable=True),
+                    Column('message', Text(), nullable=True),
+                    ForeignKeyConstraint(['task_id'], ['tasks.id'], ),
+                    PrimaryKeyConstraint('task_id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+
+def upgrade():
+    _add_tasks_table()
+    _add_task_info_table()

glance/db/sqlalchemy/alembic_migrations/alembic.ini

@@ -0,0 +1,69 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic_migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic_migrations/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# Uncomment and update to your sql connection string if wishing to run
+# alembic directly from command line
+#sqlalchemy.url =
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

glance/db/sqlalchemy/alembic_migrations/env.py

@@ -0,0 +1,92 @@
+# Copyright 2016 Rackspace
+# Copyright 2013 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from __future__ import with_statement
+from logging import config as log_config
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+from glance.db.sqlalchemy import models
+from glance.db.sqlalchemy import models_glare
+from glance.db.sqlalchemy import models_metadef
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+log_config.fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+target_metadata = models.BASE.metadata
+for table in models_glare.BASE.metadata.sorted_tables:
+    target_metadata._add_table(table.name, table.schema, table)
+for table in models_metadef.BASE_DICT.metadata.sorted_tables:
+    target_metadata._add_table(table.name, table.schema, table)
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

glance/db/sqlalchemy/alembic_migrations/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=Glance Migrations
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=alembic_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]

glance/db/sqlalchemy/alembic_migrations/script.py.mako

@@ -0,0 +1,20 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}

glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py

@@ -0,0 +1,40 @@
+# Copyright 2016 Rackspace
+# Copyright 2013 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""liberty initial
+
+Revision ID: liberty
+Revises:
+Create Date: 2016-08-03 16:06:59.657433
+
+"""
+
+from glance.db.sqlalchemy.alembic_migrations import add_artifacts_tables
+from glance.db.sqlalchemy.alembic_migrations import add_images_tables
+from glance.db.sqlalchemy.alembic_migrations import add_metadefs_tables
+from glance.db.sqlalchemy.alembic_migrations import add_tasks_tables
+
+# revision identifiers, used by Alembic.
+revision = 'liberty'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    add_images_tables.upgrade()
+    add_tasks_tables.upgrade()
+    add_metadefs_tables.upgrade()
+    add_artifacts_tables.upgrade()

glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py

@@ -0,0 +1,47 @@
+# Copyright 2016 Rackspace
+# Copyright 2013 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""add index on created_at and updated_at columns of 'images' table
+
+Revision ID: mitaka01
+Revises: liberty
+Create Date: 2016-08-03 17:19:35.306161
+
+"""
+
+from alembic import op
+from sqlalchemy import MetaData, Table, Index
+
+
+# revision identifiers, used by Alembic.
+revision = 'mitaka01'
+down_revision = 'liberty'
+branch_labels = None
+depends_on = None
+
+CREATED_AT_INDEX = 'created_at_image_idx'
+UPDATED_AT_INDEX = 'updated_at_image_idx'
+
+
+def upgrade():
+    migrate_engine = op.get_bind()
+    meta = MetaData(bind=migrate_engine)
+
+    images = Table('images', meta, autoload=True)
+
+    created_index = Index(CREATED_AT_INDEX, images.c.created_at)
+    created_index.create(migrate_engine)
+    updated_index = Index(UPDATED_AT_INDEX, images.c.updated_at)
+    updated_index.create(migrate_engine)

glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py

@@ -0,0 +1,42 @@
+# Copyright 2016 Rackspace
+# Copyright 2013 Intel Corporation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""update metadef os_nova_server
+
+Revision ID: mitaka02
+Revises: mitaka01
+Create Date: 2016-08-03 17:23:23.041663
+
+"""
+
+from alembic import op
+from sqlalchemy import MetaData, Table
+
+
+# revision identifiers, used by Alembic.
+revision = 'mitaka02'
+down_revision = 'mitaka01'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    migrate_engine = op.get_bind()
+    meta = MetaData(bind=migrate_engine)
+
+    resource_types_table = Table('metadef_resource_types', meta, autoload=True)
+
+    resource_types_table.update(values={'name': 'OS::Nova::Server'}).where(
+        resource_types_table.c.name == 'OS::Nova::Instance').execute()

glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py

@@ -0,0 +1,72 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""add visibility to and remove is_public from images
+
+Revision ID: ocata01
+Revises: mitaka02
+Create Date: 2017-01-20 12:58:16.647499
+
+"""
+
+import os
+
+from alembic import op
+from sqlalchemy import Column, Enum, MetaData, select, Table, not_, and_
+import sqlparse
+
+# revision identifiers, used by Alembic.
+revision = 'ocata01'
+down_revision = 'mitaka02'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    migrate_engine = op.get_bind()
+    meta = MetaData(bind=migrate_engine)
+
+    engine_name = migrate_engine.engine.name
+    if engine_name == 'sqlite':
+        sql_file = os.path.splitext(__file__)[0]
+        sql_file += '.sql'
+        with open(sql_file, 'r') as sqlite_script:
+            sql = sqlparse.format(sqlite_script.read(), strip_comments=True)
+            for statement in sqlparse.split(sql):
+                op.execute(statement)
+        return
+
+    enum = Enum('private', 'public', 'shared', 'community', metadata=meta,
+                name='image_visibility')
+    enum.create()
+    v_col = Column('visibility', enum, nullable=False, server_default='shared')
+    op.add_column('images', v_col)
+
+    op.create_index('visibility_image_idx', 'images', ['visibility'])
+
+    images = Table('images', meta, autoload=True)
+    images.update(values={'visibility': 'public'}).where(
+        images.c.is_public).execute()
+
+    image_members = Table('image_members', meta, autoload=True)
+
+    # NOTE(dharinic): Mark all the non-public images as 'private' first
+    images.update().values(visibility='private').where(
+        not_(images.c.is_public)).execute()
+    # NOTE(dharinic): Identify 'shared' images from the above
+    images.update().values(visibility='shared').where(and_(
+        images.c.visibility == 'private', images.c.id.in_(select(
+            [image_members.c.image_id]).distinct().where(
+                not_(image_members.c.deleted))))).execute()
+
+    op.drop_index('ix_images_is_public', 'images')
+    op.drop_column('images', 'is_public')

glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql

@@ -0,0 +1,162 @@
+CREATE TEMPORARY TABLE images_backup (
+    id VARCHAR(36) NOT NULL,
+    name VARCHAR(255),
+    size INTEGER,
+    status VARCHAR(30) NOT NULL,
+    is_public BOOLEAN NOT NULL,
+    created_at DATETIME NOT NULL,
+    updated_at DATETIME,
+    deleted_at DATETIME,
+    deleted BOOLEAN NOT NULL,
+    disk_format VARCHAR(20),
+    container_format VARCHAR(20),
+    checksum VARCHAR(32),
+    owner VARCHAR(255),
+    min_disk INTEGER NOT NULL,
+    min_ram INTEGER NOT NULL,
+    protected BOOLEAN DEFAULT 0 NOT NULL,
+    virtual_size INTEGER,
+    PRIMARY KEY (id),
+    CHECK (is_public IN (0, 1)),
+    CHECK (deleted IN (0, 1)),
+    CHECK (protected IN (0, 1))
+);
+
+INSERT INTO images_backup
+    SELECT id,
+        name,
+        size,
+        status,
+        is_public,
+        created_at,
+        updated_at,
+        deleted_at,
+        deleted,
+        disk_format,
+        container_format,
+        checksum,
+        owner,
+        min_disk,
+        min_ram,
+        protected,
+        virtual_size
+    FROM images;
+
+DROP TABLE images;
+
+CREATE TABLE images (
+    id VARCHAR(36) NOT NULL,
+    name VARCHAR(255),
+    size INTEGER,
+    status VARCHAR(30) NOT NULL,
+    created_at DATETIME NOT NULL,
+    updated_at DATETIME,
+    deleted_at DATETIME,
+    deleted BOOLEAN NOT NULL,
+    disk_format VARCHAR(20),
+    container_format VARCHAR(20),
+    checksum VARCHAR(32),
+    owner VARCHAR(255),
+    min_disk INTEGER NOT NULL,
+    min_ram INTEGER NOT NULL,
+    protected BOOLEAN DEFAULT 0 NOT NULL,
+    virtual_size INTEGER,
+    visibility VARCHAR(9) DEFAULT 'shared' NOT NULL,
+    PRIMARY KEY (id),
+    CHECK (deleted IN (0, 1)),
+    CHECK (protected IN (0, 1)),
+    CONSTRAINT image_visibility CHECK (visibility IN ('private', 'public', 'shared', 'community'))
+);
+
+CREATE INDEX checksum_image_idx ON images (checksum);
+CREATE INDEX visibility_image_idx ON images (visibility);
+CREATE INDEX ix_images_deleted ON images (deleted);
+CREATE INDEX owner_image_idx ON images (owner);
+CREATE INDEX created_at_image_idx ON images (created_at);
+CREATE INDEX updated_at_image_idx ON images (updated_at);
+
+-- Copy over all the 'public' rows
+
+INSERT INTO images (
+    id,
+    name,
+    size,
+    status,
+    created_at,
+    updated_at,
+    deleted_at,
+    deleted,
+    disk_format,
+    container_format,
+    checksum,
+    owner,
+    min_disk,
+    min_ram,
+    protected,
+    virtual_size
+    )
+    SELECT id,
+        name,
+        size,
+        status,
+        created_at,
+        updated_at,
+        deleted_at,
+        deleted,
+        disk_format,
+        container_format,
+        checksum,
+        owner,
+        min_disk,
+        min_ram,
+        protected,
+        virtual_size
+    FROM images_backup
+    WHERE is_public=1;
+
+
+UPDATE images SET visibility='public';
+
+-- Now copy over the 'private' rows
+
+INSERT INTO images (
+    id,
+    name,
+    size,
+    status,
+    created_at,
+    updated_at,
+    deleted_at,
+    deleted,
+    disk_format,
+    container_format,
+    checksum,
+    owner,
+    min_disk,
+    min_ram,
+    protected,
+    virtual_size
+    )
+    SELECT id,
+        name,
+        size,
+        status,
+        created_at,
+        updated_at,
+        deleted_at,
+        deleted,
+        disk_format,
+        container_format,
+        checksum,
+        owner,
+        min_disk,
+        min_ram,
+        protected,
+        virtual_size
+    FROM images_backup
+    WHERE is_public=0;
+
+UPDATE images SET visibility='private' WHERE visibility='shared';
+UPDATE images SET visibility='shared' WHERE visibility='private' AND id IN (SELECT DISTINCT image_id FROM image_members WHERE deleted != 1);
+
+DROP TABLE images_backup;

glance/tests/unit/test_manage.py

@@ -15,11 +15,8 @@
 
 import fixtures
 import mock
-from oslo_db.sqlalchemy import migration
-from six.moves import StringIO
 
 from glance.cmd import manage
-from glance.db import migration as db_migration
 from glance.db.sqlalchemy import api as db_api
 from glance.db.sqlalchemy import metadata as db_metadata
 from glance.tests import utils as test_utils
@@ -51,48 +48,35 @@ class TestManageBase(test_utils.BaseTestCase):
 
 class TestLegacyManage(TestManageBase):
 
-    @mock.patch.object(migration, 'db_version')
-    def test_legacy_db_version(self, db_version):
-        with mock.patch('sys.stdout', new_callable=StringIO):
-            self._main_test_helper(['glance.cmd.manage', 'db_version'],
-                                   migration.db_version,
-                                   db_api.get_engine(),
-                                   db_migration.MIGRATE_REPO_PATH, 0)
+    @mock.patch.object(manage.DbCommands, 'version')
+    def test_legacy_db_version(self, db_upgrade):
+        self._main_test_helper(['glance.cmd.manage', 'db_version'],
+                               manage.DbCommands.version)
 
-    @mock.patch.object(migration, 'db_sync')
+    @mock.patch.object(manage.DbCommands, 'sync')
     def test_legacy_db_sync(self, db_sync):
         self._main_test_helper(['glance.cmd.manage', 'db_sync'],
-                               migration.db_sync,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, None)
+                               manage.DbCommands.sync, None)
 
-    @mock.patch.object(migration, 'db_sync')
-    def test_legacy_db_upgrade(self, db_sync):
+    @mock.patch.object(manage.DbCommands, 'upgrade')
+    def test_legacy_db_upgrade(self, db_upgrade):
         self._main_test_helper(['glance.cmd.manage', 'db_upgrade'],
-                               migration.db_sync,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, None)
+                               manage.DbCommands.upgrade, None)
 
-    @mock.patch.object(migration, 'db_version_control')
+    @mock.patch.object(manage.DbCommands, 'version_control')
     def test_legacy_db_version_control(self, db_version_control):
         self._main_test_helper(['glance.cmd.manage', 'db_version_control'],
-                               migration.db_version_control,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, None)
+                               manage.DbCommands.version_control, None)
 
-    @mock.patch.object(migration, 'db_sync')
+    @mock.patch.object(manage.DbCommands, 'sync')
     def test_legacy_db_sync_version(self, db_sync):
-        self._main_test_helper(['glance.cmd.manage', 'db_sync', '20'],
-                               migration.db_sync,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, '20')
+        self._main_test_helper(['glance.cmd.manage', 'db_sync', 'liberty'],
+                               manage.DbCommands.sync, 'liberty')
 
-    @mock.patch.object(migration, 'db_sync')
-    def test_legacy_db_upgrade_version(self, db_sync):
-        self._main_test_helper(['glance.cmd.manage', 'db_upgrade', '20'],
-                               migration.db_sync,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, '20')
+    @mock.patch.object(manage.DbCommands, 'upgrade')
+    def test_legacy_db_upgrade_version(self, db_upgrade):
+        self._main_test_helper(['glance.cmd.manage', 'db_upgrade', 'liberty'],
+                               manage.DbCommands.upgrade, 'liberty')
 
     def test_db_metadefs_unload(self):
         db_metadata.db_unload_metadefs = mock.Mock()
@@ -157,48 +141,36 @@ class TestLegacyManage(TestManageBase):
 
 class TestManage(TestManageBase):
 
-    @mock.patch.object(migration, 'db_version')
-    def test_db_version(self, db_version):
-        with mock.patch('sys.stdout', new_callable=StringIO):
-            self._main_test_helper(['glance.cmd.manage', 'db', 'version'],
-                                   migration.db_version,
-                                   db_api.get_engine(),
-                                   db_migration.MIGRATE_REPO_PATH, 0)
+    @mock.patch.object(manage.DbCommands, 'version')
+    def test_db_version(self, version):
+        self._main_test_helper(['glance.cmd.manage', 'db', 'version'],
+                               manage.DbCommands.version)
 
-    @mock.patch.object(migration, 'db_sync')
-    def test_db_sync(self, db_sync):
+    @mock.patch.object(manage.DbCommands, 'sync')
+    def test_db_sync(self, sync):
         self._main_test_helper(['glance.cmd.manage', 'db', 'sync'],
-                               migration.db_sync,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, None)
+                               manage.DbCommands.sync)
 
-    @mock.patch.object(migration, 'db_sync')
-    def test_db_upgrade(self, db_sync):
+    @mock.patch.object(manage.DbCommands, 'upgrade')
+    def test_db_upgrade(self, upgrade):
         self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade'],
-                               migration.db_sync,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, None)
+                               manage.DbCommands.upgrade)
 
-    @mock.patch.object(migration, 'db_version_control')
-    def test_db_version_control(self, db_version_control):
+    @mock.patch.object(manage.DbCommands, 'version_control')
+    def test_db_version_control(self, version_control):
         self._main_test_helper(['glance.cmd.manage', 'db', 'version_control'],
-                               migration.db_version_control,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, None)
+                               manage.DbCommands.version_control)
 
-    @mock.patch.object(migration, 'db_sync')
-    def test_db_sync_version(self, db_sync):
-        self._main_test_helper(['glance.cmd.manage', 'db', 'sync', '20'],
-                               migration.db_sync,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, '20')
+    @mock.patch.object(manage.DbCommands, 'sync')
+    def test_db_sync_version(self, sync):
+        self._main_test_helper(['glance.cmd.manage', 'db', 'sync', 'liberty'],
+                               manage.DbCommands.sync, 'liberty')
 
-    @mock.patch.object(migration, 'db_sync')
-    def test_db_upgrade_version(self, db_sync):
-        self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade', '20'],
-                               migration.db_sync,
-                               db_api.get_engine(),
-                               db_migration.MIGRATE_REPO_PATH, '20')
+    @mock.patch.object(manage.DbCommands, 'upgrade')
+    def test_db_upgrade_version(self, upgrade):
+        self._main_test_helper(['glance.cmd.manage', 'db',
+                                'upgrade', 'liberty'],
+                               manage.DbCommands.upgrade, 'liberty')
 
     def test_db_metadefs_unload(self):
         db_metadata.db_unload_metadefs = mock.Mock()

requirements.txt

@@ -12,6 +12,8 @@ Routes!=2.0,!=2.1,!=2.3.0,>=1.12.3;python_version=='2.7' # MIT
 Routes!=2.0,!=2.3.0,>=1.12.3;python_version!='2.7' # MIT
 WebOb>=1.6.0 # MIT
 sqlalchemy-migrate>=0.9.6 # Apache-2.0
+sqlparse>=0.2.2 # BSD
+alembic>=0.8.10 # MIT
 httplib2>=0.7.5 # MIT
 pycrypto>=2.6 # Public Domain
 oslo.config!=3.18.0,>=3.14.0 # Apache-2.0