diff --git a/README.rst b/README.rst index b8f3130..1ef454f 100644 --- a/README.rst +++ b/README.rst @@ -27,28 +27,28 @@ DB Setup -------- The usage of subunit2ql is split into 2 stages. First you need to prepare a -database with the proper schema; alembic should be used to do this. The -alembic.ini file in-tree includes the necessary options for alembic, however -the database uri must be specificed with the option:: +database with the proper schema; subunit2sql-db-manage should be used to do +this. The utility requires db connection info which can be specified on the +command or with a config file. Obviously the sql connector type, user, +password, address, and database name should be specific to your environment. +subunit2sql-db-manage will use alembic to setup the db schema. You can run the +db migrations with the command:: - sqlalchemy.url = smysql:///user:pass@127.0.0.1/subunit + subunit2sql-db-manage --database-connection mysql://subunit:pass@127.0.0.1/subunit upgrade head -to be able to use alembic to setup the db schema. Obviously the sql connector -type, user, password, address, and database name should be specific to your -environment. After the alembic.ini file is updated you perform can run the db -migrations with the command:: +or with a config file:: - alembic upgrade head + subunit2sql-db-manage --config-file subunit2sql.conf upgrade head -from the root path for subunit2sql. This will bring the DB schema up to the -latest version for subunit2sql. Also, it is worh noting that the schema -migrations used in subunit2sql do not currently support sqlite. While it is -possible to fix this, sqlite only supports a subset of the necessary sql calls -used by the migration scripts. As such, maintaining support for sqlite will be -a continual extra effort, so if support is added in the future, it is no -guarantee that it will remain. In addition, the performance of running, even in -a testing capacity, subunit2sql with MySQL or Postgres make it worth the effort -of setting up one of them to use subunit2sql. +This will bring the DB schema up to the latest version for subunit2sql. Also, +it is worth noting that the schema migrations used in subunit2sql do not +currently support sqlite. While it is possible to fix this, sqlite only +supports a subset of the necessary sql calls used by the migration scripts. As +such, maintaining support for sqlite will be a continual extra effort, so if +support is added back in the future, it is no guarantee that it will remain. In +addition, the performance of running, even in a testing capacity, subunit2sql +with MySQL or Postgres make it worth the effort of setting up one of them to +use subunit2sql. Running subunit2sql ------------------- diff --git a/setup.cfg b/setup.cfg index 6b45b86..6266c61 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ packages = console_scripts = subunit2sql = subunit2sql.shell:main sql2subunit = subunit2sql.write_subunit:main + subunit2sql-db-manage = subunit2sql.migrations.cli:main [build_sphinx] source-dir = doc/source diff --git a/subunit2sql/migrations/__init__.py b/subunit2sql/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/alembic.ini b/subunit2sql/migrations/alembic.ini similarity index 100% rename from alembic.ini rename to subunit2sql/migrations/alembic.ini diff --git a/subunit2sql/migrations/cli.py b/subunit2sql/migrations/cli.py new file mode 100644 index 0000000..3844c06 --- /dev/null +++ b/subunit2sql/migrations/cli.py @@ -0,0 +1,152 @@ +# Copyright 2012 New Dream Network, LLC (DreamHost) +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +from alembic import command as alembic_command +from alembic import config as alembic_config +from alembic import script as alembic_script +from alembic import util as alembic_util +from oslo.config import cfg +from oslo.db import options + + +HEAD_FILENAME = 'HEAD' + + +def state_path_def(*args): + """Return an uninterpolated path relative to $state_path.""" + return os.path.join('$state_path', *args) + + +CONF = cfg.CONF +CONF.register_cli_opts(options.database_opts, group='database') + + +def do_alembic_command(config, cmd, *args, **kwargs): + try: + getattr(alembic_command, cmd)(config, *args, **kwargs) + except alembic_util.CommandError as e: + alembic_util.err(str(e)) + + +def do_check_migration(config, cmd): + do_alembic_command(config, 'branches') + validate_head_file(config) + + +def do_upgrade_downgrade(config, cmd): + if not CONF.command.revision and not CONF.command.delta: + raise SystemExit('You must provide a revision or relative delta') + + revision = CONF.command.revision + + if CONF.command.delta: + sign = '+' if CONF.command.name == 'upgrade' else '-' + revision = sign + str(CONF.command.delta) + else: + revision = CONF.command.revision + + do_alembic_command(config, cmd, revision, sql=CONF.command.sql) + + +def do_stamp(config, cmd): + do_alembic_command(config, cmd, + CONF.command.revision, + sql=CONF.command.sql) + + +def do_revision(config, cmd): + do_alembic_command(config, cmd, + message=CONF.command.message, + autogenerate=CONF.command.autogenerate, + sql=CONF.command.sql) + update_head_file(config) + + +def validate_head_file(config): + script = alembic_script.ScriptDirectory.from_config(config) + if len(script.get_heads()) > 1: + alembic_util.err('Timeline branches unable to generate timeline') + + head_path = os.path.join(script.versions, HEAD_FILENAME) + if (os.path.isfile(head_path) and + open(head_path).read().strip() == script.get_current_head()): + return + else: + alembic_util.err('HEAD file does not match migration timeline head') + + +def update_head_file(config): + script = alembic_script.ScriptDirectory.from_config(config) + if len(script.get_heads()) > 1: + alembic_util.err('Timeline branches unable to generate timeline') + + head_path = os.path.join(script.versions, HEAD_FILENAME) + with open(head_path, 'w+') as f: + f.write(script.get_current_head()) + + +def add_command_parsers(subparsers): + for name in ['current', 'history', 'branches']: + parser = subparsers.add_parser(name) + parser.set_defaults(func=do_alembic_command) + + parser = subparsers.add_parser('check_migration') + parser.set_defaults(func=do_check_migration) + + for name in ['upgrade', 'downgrade']: + parser = subparsers.add_parser(name) + parser.add_argument('--delta', type=int) + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision', nargs='?') + parser.add_argument('--mysql-engine', + default='', + help='Change MySQL storage engine of current ' + 'existing tables') + parser.set_defaults(func=do_upgrade_downgrade) + + parser = subparsers.add_parser('stamp') + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision') + parser.set_defaults(func=do_stamp) + + parser = subparsers.add_parser('revision') + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.add_argument('--sql', action='store_true') + parser.set_defaults(func=do_revision) + + +command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + +CONF.register_cli_opt(command_opt) + + +def main(): + config = alembic_config.Config(os.path.join(os.path.dirname(__file__), + 'alembic.ini')) + config.set_main_option('script_location', + 'subunit2sql:migrations') + config.subunit2sql_config = CONF + CONF() + CONF.command.func(config, CONF.command.name) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/subunit2sql/migrations/env.py b/subunit2sql/migrations/env.py index 984c42d..cc25260 100644 --- a/subunit2sql/migrations/env.py +++ b/subunit2sql/migrations/env.py @@ -15,12 +15,14 @@ from __future__ import with_statement from alembic import context -from sqlalchemy import engine_from_config, pool from logging.config import fileConfig # noqa +from oslo.db.sqlalchemy import session + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config +subunit2sql_config = config.subunit2sql_config # Interpret the config file for Python logging. # This line sets up loggers basically. @@ -50,8 +52,15 @@ def run_migrations_offline(): script output. """ - url = config.get_main_option("sqlalchemy.url") - context.configure(url=url, target_metadata=target_metadata) + kwargs = dict() + if subunit2sql_config.database.connection: + kwargs['url'] = subunit2sql_config.database.connection + elif subunit2sql_config.database.engine: + kwargs['dialect_name'] = subunit2sql_config.database.engine + else: + kwargs['url'] = config.get_main_option("sqlalchemy.url") + kwargs['target_metadata'] = target_metadata + context.configure(**kwargs) with context.begin_transaction(): context.run_migrations() @@ -64,10 +73,7 @@ def run_migrations_online(): and associate a connection with the context. """ - engine = engine_from_config(config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) - + engine = session.create_engine(subunit2sql_config.database.connection) connection = engine.connect() context.configure(connection=connection, target_metadata=target_metadata)