Add subunit2sql-db-manage utility

This commit adds a new cli utility for managing the subunit2sql db.
Previously alembic could just be used directly however since access
to the db models were needed for future migration scripts the db api
was needed. This required invoking the config object to be able to
use the db api calls. Alembic isn't aware of oslo.db or the config
object, so by creating a seperate cli interface we initialize all of
that at the same time. This also saves the need to configure
connection info to the db in 2 places, since just the oslo.db option
will be used.

This utility borrows heavily from the neutron utility to do the same
thing.

Change-Id: I110baa532d08de4ca70b7ea2d1dcdc845d595693

README.rst

@@ -27,28 +27,28 @@ DB Setup
 --------
 
 The usage of subunit2ql is split into 2 stages. First you need to prepare a
-database with the proper schema; alembic should be used to do this. The
-alembic.ini file in-tree includes the necessary options for alembic, however
-the database uri must be specificed with the option::
+database with the proper schema; subunit2sql-db-manage should be used to do
+this. The utility requires db connection info which can be specified on the
+command or with a config file. Obviously the sql connector type, user,
+password, address, and database name should be specific to your environment.
+subunit2sql-db-manage will use alembic to setup the db schema. You can run the
+db migrations with the command::
 
-    sqlalchemy.url = smysql:///user:pass@127.0.0.1/subunit
+    subunit2sql-db-manage --database-connection mysql://subunit:pass@127.0.0.1/subunit upgrade head
 
-to be able to use alembic to setup the db schema. Obviously the sql connector
-type, user, password, address, and database name should be specific to your
-environment. After the alembic.ini file is updated you perform can run the db
-migrations with the command::
+or with a config file::
 
-    alembic upgrade head
+    subunit2sql-db-manage --config-file subunit2sql.conf upgrade head
 
-from the root path for subunit2sql. This will bring the DB schema up to the
-latest version for subunit2sql. Also, it is worh noting that the schema
-migrations used in subunit2sql do not currently support sqlite. While it is
-possible to fix this, sqlite only supports a subset of the necessary sql calls
-used by the migration scripts. As such, maintaining support for sqlite will be
-a continual extra effort, so if support is added in the future, it is no
-guarantee that it will remain. In addition, the performance of running, even in
-a testing capacity, subunit2sql with MySQL or Postgres make it worth the effort
-of setting up one of them to use subunit2sql.
+This will bring the DB schema up to the latest version for subunit2sql. Also,
+it is worth noting that the schema migrations used in subunit2sql do not 
+currently support sqlite. While it is possible to fix this, sqlite only
+supports a subset of the necessary sql calls used by the migration scripts. As
+such, maintaining support for sqlite will be a continual extra effort, so if
+support is added back in the future, it is no guarantee that it will remain. In 
+addition, the performance of running, even in a testing capacity, subunit2sql
+with MySQL or Postgres make it worth the effort of setting up one of them to
+use subunit2sql.
 
 Running subunit2sql
 -------------------

setup.cfg

@@ -24,6 +24,7 @@ packages =
 console_scripts =
     subunit2sql = subunit2sql.shell:main
     sql2subunit = subunit2sql.write_subunit:main
+    subunit2sql-db-manage = subunit2sql.migrations.cli:main
 
 [build_sphinx]
 source-dir = doc/source

subunit2sql/migrations/cli.py

@@ -0,0 +1,152 @@
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import sys
+
+from alembic import command as alembic_command
+from alembic import config as alembic_config
+from alembic import script as alembic_script
+from alembic import util as alembic_util
+from oslo.config import cfg
+from oslo.db import options
+
+
+HEAD_FILENAME = 'HEAD'
+
+
+def state_path_def(*args):
+    """Return an uninterpolated path relative to $state_path."""
+    return os.path.join('$state_path', *args)
+
+
+CONF = cfg.CONF
+CONF.register_cli_opts(options.database_opts, group='database')
+
+
+def do_alembic_command(config, cmd, *args, **kwargs):
+    try:
+        getattr(alembic_command, cmd)(config, *args, **kwargs)
+    except alembic_util.CommandError as e:
+        alembic_util.err(str(e))
+
+
+def do_check_migration(config, cmd):
+    do_alembic_command(config, 'branches')
+    validate_head_file(config)
+
+
+def do_upgrade_downgrade(config, cmd):
+    if not CONF.command.revision and not CONF.command.delta:
+        raise SystemExit('You must provide a revision or relative delta')
+
+    revision = CONF.command.revision
+
+    if CONF.command.delta:
+        sign = '+' if CONF.command.name == 'upgrade' else '-'
+        revision = sign + str(CONF.command.delta)
+    else:
+        revision = CONF.command.revision
+
+    do_alembic_command(config, cmd, revision, sql=CONF.command.sql)
+
+
+def do_stamp(config, cmd):
+    do_alembic_command(config, cmd,
+                       CONF.command.revision,
+                       sql=CONF.command.sql)
+
+
+def do_revision(config, cmd):
+    do_alembic_command(config, cmd,
+                       message=CONF.command.message,
+                       autogenerate=CONF.command.autogenerate,
+                       sql=CONF.command.sql)
+    update_head_file(config)
+
+
+def validate_head_file(config):
+    script = alembic_script.ScriptDirectory.from_config(config)
+    if len(script.get_heads()) > 1:
+        alembic_util.err('Timeline branches unable to generate timeline')
+
+    head_path = os.path.join(script.versions, HEAD_FILENAME)
+    if (os.path.isfile(head_path) and
+        open(head_path).read().strip() == script.get_current_head()):
+        return
+    else:
+        alembic_util.err('HEAD file does not match migration timeline head')
+
+
+def update_head_file(config):
+    script = alembic_script.ScriptDirectory.from_config(config)
+    if len(script.get_heads()) > 1:
+        alembic_util.err('Timeline branches unable to generate timeline')
+
+    head_path = os.path.join(script.versions, HEAD_FILENAME)
+    with open(head_path, 'w+') as f:
+        f.write(script.get_current_head())
+
+
+def add_command_parsers(subparsers):
+    for name in ['current', 'history', 'branches']:
+        parser = subparsers.add_parser(name)
+        parser.set_defaults(func=do_alembic_command)
+
+    parser = subparsers.add_parser('check_migration')
+    parser.set_defaults(func=do_check_migration)
+
+    for name in ['upgrade', 'downgrade']:
+        parser = subparsers.add_parser(name)
+        parser.add_argument('--delta', type=int)
+        parser.add_argument('--sql', action='store_true')
+        parser.add_argument('revision', nargs='?')
+        parser.add_argument('--mysql-engine',
+                            default='',
+                            help='Change MySQL storage engine of current '
+                                 'existing tables')
+        parser.set_defaults(func=do_upgrade_downgrade)
+
+    parser = subparsers.add_parser('stamp')
+    parser.add_argument('--sql', action='store_true')
+    parser.add_argument('revision')
+    parser.set_defaults(func=do_stamp)
+
+    parser = subparsers.add_parser('revision')
+    parser.add_argument('-m', '--message')
+    parser.add_argument('--autogenerate', action='store_true')
+    parser.add_argument('--sql', action='store_true')
+    parser.set_defaults(func=do_revision)
+
+
+command_opt = cfg.SubCommandOpt('command',
+                                title='Command',
+                                help='Available commands',
+                                handler=add_command_parsers)
+
+CONF.register_cli_opt(command_opt)
+
+
+def main():
+    config = alembic_config.Config(os.path.join(os.path.dirname(__file__),
+                                                'alembic.ini'))
+    config.set_main_option('script_location',
+                           'subunit2sql:migrations')
+    config.subunit2sql_config = CONF
+    CONF()
+    CONF.command.func(config, CONF.command.name)
+
+if __name__ == "__main__":
+    sys.exit(main())

subunit2sql/migrations/env.py

@@ -15,12 +15,14 @@
 
 from __future__ import with_statement
 from alembic import context
-from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig  # noqa
+from oslo.db.sqlalchemy import session
+
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
+subunit2sql_config = config.subunit2sql_config
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
@@ -50,8 +52,15 @@ def run_migrations_offline():
     script output.
 
     """
-    url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url, target_metadata=target_metadata)
+    kwargs = dict()
+    if subunit2sql_config.database.connection:
+        kwargs['url'] = subunit2sql_config.database.connection
+    elif subunit2sql_config.database.engine:
+        kwargs['dialect_name'] = subunit2sql_config.database.engine
+    else:
+        kwargs['url'] = config.get_main_option("sqlalchemy.url")
+    kwargs['target_metadata'] = target_metadata
+    context.configure(**kwargs)
 
     with context.begin_transaction():
         context.run_migrations()
@@ -64,10 +73,7 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-    engine = engine_from_config(config.get_section(config.config_ini_section),
-                                prefix='sqlalchemy.',
-                                poolclass=pool.NullPool)
-
+    engine = session.create_engine(subunit2sql_config.database.connection)
     connection = engine.connect()
     context.configure(connection=connection, target_metadata=target_metadata)