Add support for sqlalchemy migration based on alembic

Implement blueprint: sqlalchemy-migration

Change-Id: Ifd0d5186299611907a388f8ab8b27e312a3cb60d

releasenotes/notes/sqlalchemy-migration-6b4eaebb6e02a449.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Add migration support for Zaqar's sqlalchemy storage driver.

requirements.txt

@@ -3,6 +3,7 @@
 # process, which may cause wedges in the gate later.
 pbr>=1.6 # Apache-2.0
 
+alembic>=0.8.4  # MIT
 Babel>=2.3.4 # BSD
 falcon>=0.1.6 # Apache-2.0
 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT

setup.cfg

@@ -32,6 +32,7 @@ console_scripts =
     zaqar-bench = zaqar.bench.conductor:main
     zaqar-server = zaqar.cmd.server:run
     zaqar-gc = zaqar.cmd.gc:run
+    zaqar-sql-db-manage = zaqar.storage.sqlalchemy.migration.cli:main
 
 zaqar.data.storage =
     mongodb = zaqar.storage.mongodb.driver:DataDriver

test-requirements.txt

@@ -19,6 +19,8 @@ fixtures>=3.0.0 # Apache-2.0/BSD
 python-subunit>=0.0.18 # Apache-2.0/BSD
 testrepository>=0.0.18 # Apache-2.0/BSD
 testtools>=1.4.0 # MIT
+oslo.db>=4.11.0,!=4.13.1,!=4.13.2  # Apache-2.0
+testresources>=0.2.4  # Apache-2.0/BSD
 
 # Documentation
 sphinx!=1.3b1,<1.4,>=1.2.1 # BSD

zaqar/storage/sqlalchemy/migration/alembic.ini

@@ -0,0 +1,54 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = zaqar/storage/sqlalchemy/migration/alembic_migrations
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+sqlalchemy.url =
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

zaqar/storage/sqlalchemy/migration/alembic_migrations/README.md

@@ -0,0 +1,73 @@
+<!--
+Copyright 2012 New Dream Network, LLC (DreamHost)
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+-->
+
+The migrations in `alembic_migrations/versions` contain the changes needed to migrate
+between Zaqar database revisions. A migration occurs by executing a script that
+details the changes needed to upgrade the database. The migration scripts
+are ordered so that multiple scripts can run sequentially. The scripts are executed by
+Zaqar's migration wrapper which uses the Alembic library to manage the migration. Zaqar
+supports migration from Liberty or later.
+
+You can upgrade to the latest database version via:
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade head
+```
+
+To check the current database version:
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf current
+```
+
+To create a script to run the migration offline:
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade head --sql
+```
+
+To run the offline migration between specific migration versions:
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade <start version>:<end version> --sql
+```
+
+Upgrade the database incrementally:
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade --delta <# of revs>
+```
+
+Create new revision:
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf revision -m "description of revision" --autogenerate
+```
+
+Create a blank file:
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf revision -m "description of revision"
+```
+
+This command does not perform any migrations, it only sets the revision.
+Revision may be any existing revision. Use this command carefully.
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf stamp <revision>
+```
+
+To verify that the timeline does branch, you can run this command:
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf check_migration
+```
+
+If the migration path does branch, you can find the branch point via:
+```
+$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf history
+```

zaqar/storage/sqlalchemy/migration/alembic_migrations/env.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Based on Neutron's migration/cli.py
+
+from __future__ import with_statement
+from logging import config as c
+
+from alembic import context
+from oslo_utils import importutils
+from sqlalchemy import create_engine
+from sqlalchemy import pool
+
+from zaqar.storage.sqlalchemy import tables
+
+
+importutils.try_import('zaqar.storage.sqlalchemy.tables')
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+zaqar_config = config.zaqar_config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+c.fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = tables.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    context.configure(
+        url=zaqar_config['drivers:management_store:sqlalchemy'].uri)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = create_engine(
+        zaqar_config['drivers:management_store:sqlalchemy'].uri,
+        poolclass=pool.NullPool)
+
+    connection = engine.connect()
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata)
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

zaqar/storage/sqlalchemy/migration/alembic_migrations/script.py.mako

@@ -0,0 +1,34 @@
+# Copyright ${create_date.year} OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}

zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py

@@ -0,0 +1,72 @@
+# Copyright 2016 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Liberty release
+
+Revision ID: 001
+Revises: None
+Create Date: 2015-09-13 20:46:25.783444
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '001'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+
+MYSQL_ENGINE = 'InnoDB'
+MYSQL_CHARSET = 'utf8'
+
+
+def upgrade():
+    op.create_table('queues',
+                    sa.Column('id', sa.INTEGER, primary_key=True),
+                    sa.Column('project', sa.String(64)),
+                    sa.Column('name', sa.String(64)),
+                    sa.Column('metadata', sa.LargeBinary),
+                    sa.UniqueConstraint('project', 'name'))
+
+    op.create_table('poolgroup',
+                    sa.Column('name', sa.String(64), primary_key=True))
+
+    op.create_table('pools',
+                    sa.Column('name', sa.String(64), primary_key=True),
+                    sa.Column('group', sa.String(64),
+                              sa.ForeignKey('poolgroup.name',
+                                            ondelete='CASCADE'),
+                              nullable=True),
+                    sa.Column('uri', sa.String(255),
+                              unique=True, nullable=False),
+                    sa.Column('weight', sa.INTEGER, nullable=False),
+                    sa.Column('options', sa.Text()))
+
+    op.create_table('flavors',
+                    sa.Column('name', sa.String(64), primary_key=True),
+                    sa.Column('project', sa.String(64)),
+                    sa.Column('pool_group', sa.String(64),
+                              sa.ForeignKey('poolgroup.name',
+                                            ondelete='CASCADE'),
+                              nullable=False),
+                    sa.Column('capabilities', sa.Text()))
+
+    op.create_table('catalogue',
+                    sa.Column('pool', sa.String(64),
+                              sa.ForeignKey('pools.name',
+                                            ondelete='CASCADE')),
+                    sa.Column('project', sa.String(64)),
+                    sa.Column('queue', sa.String(64), nullable=False),
+                    sa.UniqueConstraint('project', 'queue'))

zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/002_placeholder.py

@@ -0,0 +1,30 @@
+# Copyright 2016 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""placeholder
+
+Revision ID: 002
+Revises: 001
+Create Date: 2014-04-01 21:04:47.941098
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '002'
+down_revision = '001'
+
+
+def upgrade():
+    pass

zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/003_placeholder.py

@@ -0,0 +1,30 @@
+# Copyright 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""placeholder
+
+Revision ID: 003
+Revises: 002
+Create Date: 2014-04-01 21:05:00.270366
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '003'
+down_revision = '002'
+
+
+def upgrade():
+    pass

zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/004_placeholder.py

@@ -0,0 +1,30 @@
+# Copyright 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""placeholder
+
+Revision ID: 004
+Revises: 003
+Create Date: 2014-04-01 21:04:57.627883
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '004'
+down_revision = '003'
+
+
+def upgrade():
+    pass

zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/005_placeholder.py

@@ -0,0 +1,30 @@
+# Copyright 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""placeholder
+
+Revision ID: 005
+Revises: 004
+Create Date: 2014-04-01 21:04:54.928605
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '005'
+down_revision = '004'
+
+
+def upgrade():
+    pass

zaqar/storage/sqlalchemy/migration/cli.py

@@ -0,0 +1,118 @@
+# Copyright (c) 2016 Catalyst IT Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from alembic import command as alembic_cmd
+from alembic import config as alembic_cfg
+from alembic import util as alembic_u
+from oslo_config import cfg
+
+CONF = cfg.CONF
+
+
+def do_alembic_command(config, cmd, *args, **kwargs):
+    try:
+        getattr(alembic_cmd, cmd)(config, *args, **kwargs)
+    except alembic_u.CommandError as e:
+        alembic_u.err(str(e))
+
+
+def do_check_migration(config, _cmd):
+    do_alembic_command(config, 'branches')
+
+
+def do_upgrade_downgrade(config, cmd):
+    if not CONF.command.revision and not CONF.command.delta:
+        raise SystemExit('You must provide a revision or relative delta')
+
+    revision = CONF.command.revision
+
+    if CONF.command.delta:
+        sign = '+' if CONF.command.name == 'upgrade' else '-'
+        revision = sign + str(CONF.command.delta)
+
+    do_alembic_command(config, cmd, revision, sql=CONF.command.sql)
+
+
+def do_stamp(config, cmd):
+    do_alembic_command(config, cmd,
+                       CONF.command.revision,
+                       sql=CONF.command.sql)
+
+
+def do_revision(config, cmd):
+    do_alembic_command(config, cmd,
+                       message=CONF.command.message,
+                       autogenerate=CONF.command.autogenerate,
+                       sql=CONF.command.sql)
+
+
+def add_command_parsers(subparsers):
+    for name in ['current', 'history', 'branches']:
+        parser = subparsers.add_parser(name)
+        parser.set_defaults(func=do_alembic_command)
+
+    parser = subparsers.add_parser('check_migration')
+    parser.set_defaults(func=do_check_migration)
+
+    for name in ['upgrade', 'downgrade']:
+        parser = subparsers.add_parser(name)
+        parser.add_argument('--delta', type=int)
+        parser.add_argument('--sql', action='store_true')
+        parser.add_argument('revision', nargs='?')
+        parser.set_defaults(func=do_upgrade_downgrade)
+
+    parser = subparsers.add_parser('stamp')
+    parser.add_argument('--sql', action='store_true')
+    parser.add_argument('revision')
+    parser.set_defaults(func=do_stamp)
+
+    parser = subparsers.add_parser('revision')
+    parser.add_argument('-m', '--message')
+    parser.add_argument('--autogenerate', action='store_true')
+    parser.add_argument('--sql', action='store_true')
+    parser.set_defaults(func=do_revision)
+
+
+command_opt = cfg.SubCommandOpt('command',
+                                title='Command',
+                                help='Available commands',
+                                handler=add_command_parsers)
+
+CONF.register_cli_opt(command_opt)
+
+sqlalchemy_opts = [cfg.StrOpt('uri',
+                              help='The SQLAlchemy connection string to'
+                              ' use to connect to the database.',
+                              secret=True)]
+
+CONF.register_opts(sqlalchemy_opts,
+                   group='drivers:management_store:sqlalchemy')
+
+
+def main():
+    config = alembic_cfg.Config(
+        os.path.join(os.path.dirname(__file__), 'alembic.ini')
+    )
+    config.set_main_option('script_location',
+                           'zaqar.storage.sqlalchemy.'
+                           'migration:alembic_migrations')
+
+    # attach the octavia conf to the Alembic conf
+    config.zaqar_config = CONF
+
+    CONF(project='zaqar')
+    CONF.command.func(config, CONF.command.name)

zaqar/tests/unit/storage/sqlalchemy_migration/test_db_manage_cli.py

@@ -0,0 +1,89 @@
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+import mock
+import testscenarios
+import testtools
+
+from zaqar.storage.sqlalchemy.migration import cli
+
+
+class TestCli(testtools.TestCase):
+    func_name = ''
+    exp_args = ()
+    exp_kwargs = {}
+
+    scenarios = [
+        ('stamp',
+         dict(argv=['prog', 'stamp', 'foo'], func_name='stamp',
+              exp_args=('foo',), exp_kwargs={'sql': False})),
+        ('stamp-sql',
+         dict(argv=['prog', 'stamp', 'foo', '--sql'], func_name='stamp',
+              exp_args=('foo',), exp_kwargs={'sql': True})),
+        ('current',
+         dict(argv=['prog', 'current'], func_name='current',
+              exp_args=[], exp_kwargs=dict())),
+        ('history',
+         dict(argv=['prog', 'history'], func_name='history',
+              exp_args=[], exp_kwargs=dict())),
+        ('check_migration',
+         dict(argv=['prog', 'check_migration'], func_name='branches',
+              exp_args=[], exp_kwargs=dict())),
+        ('sync_revision_autogenerate',
+         dict(argv=['prog', 'revision', '--autogenerate', '-m', 'message'],
+              func_name='revision',
+              exp_args=(),
+              exp_kwargs={
+                  'message': 'message', 'sql': False, 'autogenerate': True})),
+        ('sync_revision_sql',
+         dict(argv=['prog', 'revision', '--sql', '-m', 'message'],
+              func_name='revision',
+              exp_args=(),
+              exp_kwargs={
+                  'message': 'message', 'sql': True, 'autogenerate': False})),
+        ('upgrade-sql',
+         dict(argv=['prog', 'upgrade', '--sql', 'head'],
+              func_name='upgrade',
+              exp_args=('head',),
+              exp_kwargs={'sql': True})),
+
+        ('upgrade-delta',
+         dict(argv=['prog', 'upgrade', '--delta', '3'],
+              func_name='upgrade',
+              exp_args=('+3',),
+              exp_kwargs={'sql': False}))
+    ]
+
+    def setUp(self):
+        super(TestCli, self).setUp()
+        do_alembic_cmd_p = mock.patch.object(cli, 'do_alembic_command')
+        self.addCleanup(do_alembic_cmd_p.stop)
+        self.do_alembic_cmd = do_alembic_cmd_p.start()
+        self.addCleanup(cli.CONF.reset)
+
+    def test_cli(self):
+        with mock.patch.object(sys, 'argv', self.argv):
+            cli.main()
+            self.do_alembic_cmd.assert_has_calls(
+                [mock.call(
+                    mock.ANY, self.func_name,
+                    *self.exp_args, **self.exp_kwargs)]
+            )
+
+
+def load_tests(loader, in_tests, pattern):
+    return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)

zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py

@@ -0,0 +1,175 @@
+# Copyright 2014 OpenStack Foundation
+# Copyright 2014 Mirantis Inc
+# Copyright 2016 Catalyst IT Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Tests for database migrations.
+
+For the opportunistic testing you need to set up a db named 'openstack_citest'
+with user 'openstack_citest' and password 'openstack_citest' on localhost.
+The test will then use that db and u/p combo to run the tests.
+
+For postgres on Ubuntu this can be done with the following commands:
+
+sudo -u postgres psql
+postgres=# create user openstack_citest with createdb login password
+      'openstack_citest';
+postgres=# create database openstack_citest with owner openstack_citest;
+
+"""
+
+import os
+
+from oslo_db.sqlalchemy import test_base
+from oslo_db.sqlalchemy import utils as db_utils
+
+from zaqar.tests.unit.storage.sqlalchemy_migration import \
+    test_migrations_base as base
+
+
+class ZaqarMigrationsCheckers(object):
+
+    def assertColumnExists(self, engine, table, column):
+        t = db_utils.get_table(engine, table)
+        self.assertIn(column, t.c)
+
+    def assertColumnsExist(self, engine, table, columns):
+        for column in columns:
+            self.assertColumnExists(engine, table, column)
+
+    def assertColumnType(self, engine, table, column, column_type):
+        t = db_utils.get_table(engine, table)
+        column_ref_type = str(t.c[column].type)
+        self.assertEqual(column_ref_type, column_type)
+
+    def assertColumnCount(self, engine, table, columns):
+        t = db_utils.get_table(engine, table)
+        self.assertEqual(len(columns), len(t.columns))
+
+    def assertColumnNotExists(self, engine, table, column):
+        t = db_utils.get_table(engine, table)
+        self.assertNotIn(column, t.c)
+
+    def assertIndexExists(self, engine, table, index):
+        t = db_utils.get_table(engine, table)
+        index_names = [idx.name for idx in t.indexes]
+        self.assertIn(index, index_names)
+
+    def assertIndexMembers(self, engine, table, index, members):
+        self.assertIndexExists(engine, table, index)
+
+        t = db_utils.get_table(engine, table)
+        index_columns = None
+        for idx in t.indexes:
+            if idx.name == index:
+                index_columns = idx.columns.keys()
+                break
+
+        self.assertEqual(sorted(members), sorted(index_columns))
+
+    def test_walk_versions(self):
+        self.walk_versions(self.engine)
+
+    def _pre_upgrade_001(self, engine):
+        # Anything returned from this method will be
+        # passed to corresponding _check_xxx method as 'data'.
+        pass
+
+    def _check_001(self, engine, data):
+        queues_columns = [
+            'id',
+            'name',
+            'project',
+            'metadata'
+        ]
+        self.assertColumnsExist(
+            engine, 'queues', queues_columns)
+        self.assertColumnCount(
+            engine, 'queues', queues_columns)
+
+        poolgroup_columns = [
+            'name',
+        ]
+        self.assertColumnsExist(
+            engine, 'poolgroup', poolgroup_columns)
+        self.assertColumnCount(
+            engine, 'poolgroup', poolgroup_columns)
+
+        pools_columns = [
+            'name',
+            'group',
+            'uri',
+            'weight',
+            'options',
+        ]
+        self.assertColumnsExist(
+            engine, 'pools', pools_columns)
+        self.assertColumnCount(
+            engine, 'pools', pools_columns)
+
+        flavors_columns = [
+            'name',
+            'project',
+            'pool_group',
+            'capabilities',
+        ]
+        self.assertColumnsExist(
+            engine, 'flavors', flavors_columns)
+        self.assertColumnCount(
+            engine, 'flavors', flavors_columns)
+
+        catalogue_columns = [
+            'pool',
+            'project',
+            'queue',
+        ]
+        self.assertColumnsExist(
+            engine, 'catalogue', catalogue_columns)
+        self.assertColumnCount(
+            engine, 'catalogue', catalogue_columns)
+
+        self._data_001(engine, data)
+
+    def _data_001(self, engine, data):
+        datasize = 512 * 1024  # 512kB
+        data = os.urandom(datasize)
+        t = db_utils.get_table(engine, 'job_binary_internal')
+        engine.execute(t.insert(), data=data, id='123', name='name')
+        new_data = engine.execute(t.select()).fetchone().data
+        self.assertEqual(data, new_data)
+        engine.execute(t.delete())
+
+    def _check_002(self, engine, data):
+        # currently, 002 is just a placeholder
+        pass
+
+    def _check_003(self, engine, data):
+        # currently, 003 is just a placeholder
+        pass
+
+    def _check_004(self, engine, data):
+        # currently, 004 is just a placeholder
+        pass
+
+    def _check_005(self, engine, data):
+        # currently, 005 is just a placeholder
+        pass
+
+
+class TestMigrationsMySQL(ZaqarMigrationsCheckers,
+                          base.BaseWalkMigrationTestCase,
+                          base.TestModelsMigrationsSync,
+                          test_base.MySQLOpportunisticTestCase):
+    pass

zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py

@@ -0,0 +1,188 @@
+# Copyright 2010-2011 OpenStack Foundation
+# Copyright 2012-2013 IBM Corp.
+# Copyright 2016 Catalyst IT Ltd.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+#
+# Ripped off from Nova's test_migrations.py
+# The only difference between Nova and this code is usage of alembic instead
+# of sqlalchemy migrations.
+#
+# There is an ongoing work to extact similar code to oslo incubator. Once it is
+# extracted we'll be able to remove this file and use oslo.
+
+import io
+import os
+import sqlalchemy as sa
+
+import alembic
+from alembic import command
+from alembic import config as alembic_config
+from alembic import migration
+from alembic import script as alembic_script
+from oslo_config import cfg
+from oslo_db.sqlalchemy import test_migrations as t_m
+from oslo_log import log as logging
+
+from zaqar.i18n import _LE
+import zaqar.storage.sqlalchemy.migration
+from zaqar.storage.sqlalchemy import tables
+
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+class BaseWalkMigrationTestCase(object):
+
+    ALEMBIC_CONFIG = alembic_config.Config(
+        os.path.join(
+            os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__),
+            'alembic.ini')
+    )
+
+    ALEMBIC_CONFIG.zaqar_config = CONF
+
+    def _configure(self, engine):
+        """For each type of repository we should do some of configure steps.
+
+        For migrate_repo we should set under version control our database.
+        For alembic we should configure database settings. For this goal we
+        should use oslo_config and openstack.commom.db.sqlalchemy.session with
+        database functionality (reset default settings and session cleanup).
+        """
+        CONF.set_override('uri', str(engine.url),
+                          group='drivers:management_store:sqlalchemy',
+                          enforce_type=True)
+        sa.cleanup()
+
+    def _alembic_command(self, alembic_command, engine, *args, **kwargs):
+        """Most of alembic command return data into output.
+
+        We should redefine this setting for getting info.
+        """
+        self.ALEMBIC_CONFIG.stdout = buf = io.StringIO()
+        CONF.set_override('uri', str(engine.url),
+                          group='drivers:management_store:sqlalchemy',
+                          enforce_type=True)
+        sa.cleanup()
+        getattr(command, alembic_command)(*args, **kwargs)
+        res = buf.getvalue().strip()
+        LOG.debug('Alembic command {command} returns: {result}'.format(
+                  command=alembic_command, result=res))
+        sa.cleanup()
+        return res
+
+    def _get_versions(self):
+        """Stores a list of versions.
+
+        Since alembic version has a random algorithm of generation
+        (SA-migrate has an ordered autoincrement naming) we should store
+        a list of versions (version for upgrade)
+        for successful testing of migrations in up mode.
+        """
+
+        env = alembic_script.ScriptDirectory.from_config(self.ALEMBIC_CONFIG)
+        versions = []
+        for rev in env.walk_revisions():
+            versions.append(rev.revision)
+
+        versions.reverse()
+        return versions
+
+    def walk_versions(self, engine=None):
+        # Determine latest version script from the repo, then
+        # upgrade from 1 through to the latest, with no data
+        # in the databases. This just checks that the schema itself
+        # upgrades successfully.
+
+        self._configure(engine)
+        versions = self._get_versions()
+        for ver in versions:
+            self._migrate_up(engine, ver, with_data=True)
+
+    def _get_version_from_db(self, engine):
+        """Returns latest version from db for each type of migrate repo."""
+
+        conn = engine.connect()
+        try:
+            context = migration.MigrationContext.configure(conn)
+            version = context.get_current_revision() or '-1'
+        finally:
+            conn.close()
+        return version
+
+    def _migrate(self, engine, version, cmd):
+        """Base method for manipulation with migrate repo.
+
+        It will upgrade or downgrade the actual database.
+        """
+
+        self._alembic_command(cmd, engine, self.ALEMBIC_CONFIG, version)
+
+    def _migrate_up(self, engine, version, with_data=False):
+        """migrate up to a new version of the db.
+
+        We allow for data insertion and post checks at every
+        migration version with special _pre_upgrade_### and
+        _check_### functions in the main test.
+        """
+        # NOTE(sdague): try block is here because it's impossible to debug
+        # where a failed data migration happens otherwise
+        check_version = version
+        try:
+            if with_data:
+                data = None
+                pre_upgrade = getattr(
+                    self, "_pre_upgrade_%s" % check_version, None)
+                if pre_upgrade:
+                    data = pre_upgrade(engine)
+            self._migrate(engine, version, 'upgrade')
+            self.assertEqual(version, self._get_version_from_db(engine))
+            if with_data:
+                check = getattr(self, "_check_%s" % check_version, None)
+                if check:
+                    check(engine, data)
+        except Exception:
+            LOG.error(_LE("Failed to migrate to version {version} on engine "
+                      "{engine}").format(version=version, engine=engine))
+            raise
+
+
+class TestModelsMigrationsSync(t_m.ModelsMigrationsSync):
+    """Class for comparison of DB migration scripts and models.
+
+    Allows to check if the DB schema obtained by applying of migration
+    scripts is equal to the one produced from models definitions.
+    """
+
+    ALEMBIC_CONFIG = alembic_config.Config(
+        os.path.join(
+            os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__),
+            'alembic.ini')
+    )
+    ALEMBIC_CONFIG.zaqar_config = CONF
+
+    def get_engine(self):
+        return self.engine
+
+    def db_sync(self, engine):
+        CONF.set_override('uri', str(engine.url),
+                          group='drivers:management_store:sqlalchemy',
+                          enforce_type=True)
+        alembic.command.upgrade(self.ALEMBIC_CONFIG, 'head')
+
+    def get_metadata(self):
+        return tables.metadata