Implements basic migration mechanism for extensions

Each extension specific table in the database
should have extension specific prefix.
For each extension configure different version_table,
which is used by alembic to store information about
current state of the database.

* extended manage.py in order to work with extensions
  migrations
* added basic migration structure for volume_manager
  extension

To be done in separate patches:
* fix syncdb to run migrations for core and for
  all extensions
* move the data about volumes from node attributes
  into extension models

Implements blueprint: volume-manager-refactoring

Change-Id: I6b433d69dc9f6f7e690e66751d06f052d17db2b6

nailgun/manage.py

@@ -154,6 +154,13 @@ def load_settings_parsers(subparsers):
     )
 
 
+def load_extensions_parsers(subparsers):
+    extensions_parser = subparsers.add_parser(
+        'extensions', help='extensions related actions')
+
+    load_alembic_parsers(extensions_parser)
+
+
 def action_dumpdata(params):
     import logging
 
@@ -212,8 +219,24 @@ def action_dropdb(params):
 
 
 def action_migrate(params):
-    from nailgun.db.migration import action_migrate_alembic
-    action_migrate_alembic(params)
+    from nailgun.db.migration import action_migrate_alembic_core
+    action_migrate_alembic_core(params)
+
+
+def action_extensions(params):
+    from nailgun.logger import logger
+    from nailgun.db.migration import action_migrate_alembic_extension
+    from nailgun.extensions import get_all_extensions
+
+    for extension in get_all_extensions():
+        if extension.alembic_migrations_path():
+            logger.info('Running command for extension {0}'.format(
+                extension.full_name()))
+            action_migrate_alembic_extension(params, extension=extension)
+        else:
+            logger.info(
+                'Extension {0} does not have migrations. '
+                'Skipping...'.format(extension.full_name()))
 
 
 def action_test(params):
@@ -302,6 +325,7 @@ if __name__ == "__main__":
     load_test_parsers(subparsers)
     load_shell_parsers(subparsers)
     load_settings_parsers(subparsers)
+    load_extensions_parsers(subparsers)
 
     params, other_params = parser.parse_known_args()
     sys.argv.pop(1)

nailgun/nailgun/db/migration/__init__.py

@@ -23,45 +23,48 @@ from alembic import util as alembic_util
 from nailgun.db.sqlalchemy import db_str
 
 
-ALEMBIC_CONFIG = alembic_config.Config(
-    os.path.join(os.path.dirname(__file__), 'alembic.ini')
-)
-ALEMBIC_CONFIG.set_main_option(
-    'script_location',
-    'nailgun.db.migration:alembic_migrations'
-)
-ALEMBIC_CONFIG.set_main_option(
-    'sqlalchemy.url',
-    db_str
-)
+def make_alembic_config(script_location, version_table):
+    config = alembic_config.Config(
+        os.path.join(os.path.dirname(__file__), 'alembic.ini'))
+    config.set_main_option('script_location', script_location)
+    config.set_main_option('sqlalchemy.url', db_str)
+    config.set_main_option('version_table', version_table)
+
+    return config
 
 
-def do_alembic_command(cmd, *args, **kwargs):
+# Alembic config for core components
+ALEMBIC_CONFIG = make_alembic_config(
+    'nailgun.db.migration:alembic_migrations',
+    'alembic_version')
+
+
+def do_alembic_command(cmd, config, *args, **kwargs):
     try:
-        getattr(alembic_command, cmd)(ALEMBIC_CONFIG, *args, **kwargs)
+        getattr(alembic_command, cmd)(config, *args, **kwargs)
     except alembic_util.CommandError as e:
         alembic_util.err(str(e))
 
 
-def do_stamp(cmd):
+def do_stamp(cmd, config):
     do_alembic_command(
         cmd,
-        ALEMBIC_CONFIG.params.revision,
-        sql=ALEMBIC_CONFIG.params.sql
-    )
+        config,
+        config.params.revision,
+        sql=config.params.sql)
 
 
-def do_revision(cmd):
+def do_revision(cmd, config):
     do_alembic_command(
         cmd,
-        message=ALEMBIC_CONFIG.params.message,
-        autogenerate=ALEMBIC_CONFIG.params.autogenerate,
-        sql=ALEMBIC_CONFIG.params.sql
-    )
+        config,
+        message=config.params.message,
+        autogenerate=config.params.autogenerate,
+        sql=config.params.sql)
 
 
-def do_upgrade_downgrade(cmd):
-    params = ALEMBIC_CONFIG.params
+def do_upgrade_downgrade(cmd, config):
+    params = config.params
     if not params.revision and not params.delta:
         raise SystemExit('You must provide a revision or relative delta')
 
@@ -71,17 +74,14 @@ def do_upgrade_downgrade(cmd):
     else:
         revision = params.revision
 
-    do_alembic_command(cmd, revision, sql=params.sql)
+    do_alembic_command(cmd, config, revision, sql=params.sql)
 
 
-def do_upgrade_head():
-    do_alembic_command("upgrade", "head")
+def do_upgrade_head(config=ALEMBIC_CONFIG):
+    do_alembic_command("upgrade", config, "head")
 
 
-def action_migrate_alembic(params):
-    global ALEMBIC_CONFIG
-    ALEMBIC_CONFIG.params = params
-
+def action_migrate(config):
     actions = {
         'current': do_alembic_command,
         'history': do_alembic_command,
@@ -92,7 +92,25 @@ def action_migrate_alembic(params):
         'revision': do_revision
     }
 
-    actions[params.alembic_command](params.alembic_command)
+    actions[config.params.alembic_command](
+        config.params.alembic_command, config)
+
+
+def action_migrate_alembic_core(params):
+    global ALEMBIC_CONFIG
+    config = ALEMBIC_CONFIG
+    config.params = params
+
+    action_migrate(config)
+
+
+def action_migrate_alembic_extension(params, extension):
+    config = make_alembic_config(
+        extension.alembic_migrations_path(),
+        extension.alembic_table_version())
+
+    config.params = params
+    action_migrate(config)
 
 
 def drop_migration_meta(engine):

nailgun/nailgun/extensions/base.py

@@ -14,7 +14,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-
 import abc
 import six
 
@@ -42,6 +41,13 @@ class BaseExtension(object):
     # ]
     urls = []
 
+    @classmethod
+    def alembic_migrations_path(cls):
+        """If extension provides database migrations,
+        the method should return path to alembic migrations
+        """
+        return None
+
     @abc.abstractproperty
     def name(self):
         """Uniq name of the extension."""
@@ -51,3 +57,17 @@ class BaseExtension(object):
         """Version of the extension, follow semantic
         versioning schema (http://semver.org/)
         """
+
+    @classmethod
+    def full_name(cls):
+        """Returns extension's name and version in human readable format
+        """
+        return '{0}-{1}'.format(cls.name, cls.version)
+
+    @classmethod
+    def table_prefix(cls):
+        return '{0}_{1}_'.format(cls.name, cls.version)
+
+    @classmethod
+    def alembic_table_version(cls):
+        return '{0}alembic_version'.format(cls.table_prefix())

nailgun/nailgun/extensions/volume_manager/alembic_migrations/migrations/env.py

@@ -0,0 +1,90 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from __future__ import with_statement
+
+from alembic import context
+from logging.config import fileConfig
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    context.configure(
+        url=config.get_main_option('sqlalchemy.url'),
+        version_table=config.get_main_option('version_table'))
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+    connection = engine.connect()
+
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata,
+        version_table=config.get_main_option('version_table'))
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

nailgun/nailgun/extensions/volume_manager/alembic_migrations/migrations/script.py.mako

@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

nailgun/nailgun/extensions/volume_manager/extension.py

@@ -14,6 +14,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
 
 from .handlers.disks import NodeDefaultsDisksHandler
 from .handlers.disks import NodeDisksHandler
@@ -27,6 +28,11 @@ class VolumeManagerExtension(BaseExtension):
     name = 'volume_manager'
     version = '1.0.0'
 
+    @classmethod
+    def alembic_migrations_path(cls):
+        return os.path.join(os.path.dirname(__file__),
+                            'alembic_migrations', 'migrations')
+
     urls = [
         {'uri': r'/nodes/(?P<node_id>\d+)/disks/?$',
          'handler': NodeDisksHandler},

nailgun/nailgun/test/unit/test_extensions.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nailgun.extensions import BaseExtension
+from nailgun.test.base import BaseTestCase
+
+
+class TestBaseExtension(BaseTestCase):
+
+    def setUp(self):
+        super(TestBaseExtension, self).setUp()
+
+        class Extexnsion(BaseExtension):
+            name = 'ext_name'
+            version = '1.0.0'
+
+        self.extension = Extexnsion()
+
+    def test_alembic_table_version(self):
+        self.assertEqual(
+            self.extension.alembic_table_version(),
+            'ext_name_1.0.0_alembic_version')
+
+    def test_table_prefix(self):
+        self.assertEqual(
+            self.extension.table_prefix(),
+            'ext_name_1.0.0_')
+
+    def test_alembic_migrations_path_none_by_default(self):
+        self.assertIsNone(self.extension.alembic_migrations_path())
+
+    def test_full_name(self):
+        self.assertEqual(
+            self.extension.full_name(),
+            'ext_name-1.0.0')