From 1394000b39b8f1aa6f3bf394f45eac227f0b7e46 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Tue, 2 Sep 2014 13:44:00 -0400 Subject: [PATCH] Add subunit2sql-db-manage utility This commit adds a new cli utility for managing the subunit2sql db. Previously alembic could just be used directly however since access to the db models were needed for future migration scripts the db api was needed. This required invoking the config object to be able to use the db api calls. Alembic isn't aware of oslo.db or the config object, so by creating a seperate cli interface we initialize all of that at the same time. This also saves the need to configure connection info to the db in 2 places, since just the oslo.db option will be used. This utility borrows heavily from the neutron utility to do the same thing. Change-Id: I110baa532d08de4ca70b7ea2d1dcdc845d595693 --- README.rst | 36 ++--- setup.cfg | 1 + subunit2sql/migrations/__init__.py | 0 .../migrations/alembic.ini | 0 subunit2sql/migrations/cli.py | 152 ++++++++++++++++++ subunit2sql/migrations/env.py | 20 ++- 6 files changed, 184 insertions(+), 25 deletions(-) create mode 100644 subunit2sql/migrations/__init__.py rename alembic.ini => subunit2sql/migrations/alembic.ini (100%) create mode 100644 subunit2sql/migrations/cli.py diff --git a/README.rst b/README.rst index b8f3130..1ef454f 100644 --- a/README.rst +++ b/README.rst @@ -27,28 +27,28 @@ DB Setup -------- The usage of subunit2ql is split into 2 stages. First you need to prepare a -database with the proper schema; alembic should be used to do this. The -alembic.ini file in-tree includes the necessary options for alembic, however -the database uri must be specificed with the option:: +database with the proper schema; subunit2sql-db-manage should be used to do +this. The utility requires db connection info which can be specified on the +command or with a config file. Obviously the sql connector type, user, +password, address, and database name should be specific to your environment. +subunit2sql-db-manage will use alembic to setup the db schema. You can run the +db migrations with the command:: - sqlalchemy.url = smysql:///user:pass@127.0.0.1/subunit + subunit2sql-db-manage --database-connection mysql://subunit:pass@127.0.0.1/subunit upgrade head -to be able to use alembic to setup the db schema. Obviously the sql connector -type, user, password, address, and database name should be specific to your -environment. After the alembic.ini file is updated you perform can run the db -migrations with the command:: +or with a config file:: - alembic upgrade head + subunit2sql-db-manage --config-file subunit2sql.conf upgrade head -from the root path for subunit2sql. This will bring the DB schema up to the -latest version for subunit2sql. Also, it is worh noting that the schema -migrations used in subunit2sql do not currently support sqlite. While it is -possible to fix this, sqlite only supports a subset of the necessary sql calls -used by the migration scripts. As such, maintaining support for sqlite will be -a continual extra effort, so if support is added in the future, it is no -guarantee that it will remain. In addition, the performance of running, even in -a testing capacity, subunit2sql with MySQL or Postgres make it worth the effort -of setting up one of them to use subunit2sql. +This will bring the DB schema up to the latest version for subunit2sql. Also, +it is worth noting that the schema migrations used in subunit2sql do not +currently support sqlite. While it is possible to fix this, sqlite only +supports a subset of the necessary sql calls used by the migration scripts. As +such, maintaining support for sqlite will be a continual extra effort, so if +support is added back in the future, it is no guarantee that it will remain. In +addition, the performance of running, even in a testing capacity, subunit2sql +with MySQL or Postgres make it worth the effort of setting up one of them to +use subunit2sql. Running subunit2sql ------------------- diff --git a/setup.cfg b/setup.cfg index 6b45b86..6266c61 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ packages = console_scripts = subunit2sql = subunit2sql.shell:main sql2subunit = subunit2sql.write_subunit:main + subunit2sql-db-manage = subunit2sql.migrations.cli:main [build_sphinx] source-dir = doc/source diff --git a/subunit2sql/migrations/__init__.py b/subunit2sql/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/alembic.ini b/subunit2sql/migrations/alembic.ini similarity index 100% rename from alembic.ini rename to subunit2sql/migrations/alembic.ini diff --git a/subunit2sql/migrations/cli.py b/subunit2sql/migrations/cli.py new file mode 100644 index 0000000..3844c06 --- /dev/null +++ b/subunit2sql/migrations/cli.py @@ -0,0 +1,152 @@ +# Copyright 2012 New Dream Network, LLC (DreamHost) +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +from alembic import command as alembic_command +from alembic import config as alembic_config +from alembic import script as alembic_script +from alembic import util as alembic_util +from oslo.config import cfg +from oslo.db import options + + +HEAD_FILENAME = 'HEAD' + + +def state_path_def(*args): + """Return an uninterpolated path relative to $state_path.""" + return os.path.join('$state_path', *args) + + +CONF = cfg.CONF +CONF.register_cli_opts(options.database_opts, group='database') + + +def do_alembic_command(config, cmd, *args, **kwargs): + try: + getattr(alembic_command, cmd)(config, *args, **kwargs) + except alembic_util.CommandError as e: + alembic_util.err(str(e)) + + +def do_check_migration(config, cmd): + do_alembic_command(config, 'branches') + validate_head_file(config) + + +def do_upgrade_downgrade(config, cmd): + if not CONF.command.revision and not CONF.command.delta: + raise SystemExit('You must provide a revision or relative delta') + + revision = CONF.command.revision + + if CONF.command.delta: + sign = '+' if CONF.command.name == 'upgrade' else '-' + revision = sign + str(CONF.command.delta) + else: + revision = CONF.command.revision + + do_alembic_command(config, cmd, revision, sql=CONF.command.sql) + + +def do_stamp(config, cmd): + do_alembic_command(config, cmd, + CONF.command.revision, + sql=CONF.command.sql) + + +def do_revision(config, cmd): + do_alembic_command(config, cmd, + message=CONF.command.message, + autogenerate=CONF.command.autogenerate, + sql=CONF.command.sql) + update_head_file(config) + + +def validate_head_file(config): + script = alembic_script.ScriptDirectory.from_config(config) + if len(script.get_heads()) > 1: + alembic_util.err('Timeline branches unable to generate timeline') + + head_path = os.path.join(script.versions, HEAD_FILENAME) + if (os.path.isfile(head_path) and + open(head_path).read().strip() == script.get_current_head()): + return + else: + alembic_util.err('HEAD file does not match migration timeline head') + + +def update_head_file(config): + script = alembic_script.ScriptDirectory.from_config(config) + if len(script.get_heads()) > 1: + alembic_util.err('Timeline branches unable to generate timeline') + + head_path = os.path.join(script.versions, HEAD_FILENAME) + with open(head_path, 'w+') as f: + f.write(script.get_current_head()) + + +def add_command_parsers(subparsers): + for name in ['current', 'history', 'branches']: + parser = subparsers.add_parser(name) + parser.set_defaults(func=do_alembic_command) + + parser = subparsers.add_parser('check_migration') + parser.set_defaults(func=do_check_migration) + + for name in ['upgrade', 'downgrade']: + parser = subparsers.add_parser(name) + parser.add_argument('--delta', type=int) + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision', nargs='?') + parser.add_argument('--mysql-engine', + default='', + help='Change MySQL storage engine of current ' + 'existing tables') + parser.set_defaults(func=do_upgrade_downgrade) + + parser = subparsers.add_parser('stamp') + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision') + parser.set_defaults(func=do_stamp) + + parser = subparsers.add_parser('revision') + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.add_argument('--sql', action='store_true') + parser.set_defaults(func=do_revision) + + +command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + +CONF.register_cli_opt(command_opt) + + +def main(): + config = alembic_config.Config(os.path.join(os.path.dirname(__file__), + 'alembic.ini')) + config.set_main_option('script_location', + 'subunit2sql:migrations') + config.subunit2sql_config = CONF + CONF() + CONF.command.func(config, CONF.command.name) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/subunit2sql/migrations/env.py b/subunit2sql/migrations/env.py index 984c42d..cc25260 100644 --- a/subunit2sql/migrations/env.py +++ b/subunit2sql/migrations/env.py @@ -15,12 +15,14 @@ from __future__ import with_statement from alembic import context -from sqlalchemy import engine_from_config, pool from logging.config import fileConfig # noqa +from oslo.db.sqlalchemy import session + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config +subunit2sql_config = config.subunit2sql_config # Interpret the config file for Python logging. # This line sets up loggers basically. @@ -50,8 +52,15 @@ def run_migrations_offline(): script output. """ - url = config.get_main_option("sqlalchemy.url") - context.configure(url=url, target_metadata=target_metadata) + kwargs = dict() + if subunit2sql_config.database.connection: + kwargs['url'] = subunit2sql_config.database.connection + elif subunit2sql_config.database.engine: + kwargs['dialect_name'] = subunit2sql_config.database.engine + else: + kwargs['url'] = config.get_main_option("sqlalchemy.url") + kwargs['target_metadata'] = target_metadata + context.configure(**kwargs) with context.begin_transaction(): context.run_migrations() @@ -64,10 +73,7 @@ def run_migrations_online(): and associate a connection with the context. """ - engine = engine_from_config(config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) - + engine = session.create_engine(subunit2sql_config.database.connection) connection = engine.connect() context.configure(connection=connection, target_metadata=target_metadata)