From 2d3cbedc89402ed2ef7db285f5fc97f23c2a8d77 Mon Sep 17 00:00:00 2001 From: Masayuki Igawa Date: Tue, 19 Apr 2016 19:00:55 +0900 Subject: [PATCH] Add migration and shell --- .gitignore | 2 + coverage2sql/db/__init__.py | 0 coverage2sql/db/api.py | 70 +++++++ coverage2sql/db/models.py | 58 ++++++ coverage2sql/migrations/README | 1 + coverage2sql/migrations/cli.py | 186 ++++++++++++++++++ coverage2sql/migrations/env.py | 85 ++++++++ coverage2sql/migrations/script.py.mako | 24 +++ .../52dfb338f74e_add_coverages_table.py | 31 +++ coverage2sql/shell.py | 76 +++++++ requirements.txt | 3 + setup.cfg | 12 +- test-requirements.txt | 1 + 13 files changed, 548 insertions(+), 1 deletion(-) create mode 100644 coverage2sql/db/__init__.py create mode 100644 coverage2sql/db/api.py create mode 100644 coverage2sql/db/models.py create mode 100644 coverage2sql/migrations/README create mode 100644 coverage2sql/migrations/cli.py create mode 100644 coverage2sql/migrations/env.py create mode 100644 coverage2sql/migrations/script.py.mako create mode 100644 coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py create mode 100644 coverage2sql/shell.py diff --git a/.gitignore b/.gitignore index e52bb44..c6f93e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +alembic.ini + *.py[cod] # C extensions diff --git a/coverage2sql/db/__init__.py b/coverage2sql/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/coverage2sql/db/api.py b/coverage2sql/db/api.py new file mode 100644 index 0000000..5ade229 --- /dev/null +++ b/coverage2sql/db/api.py @@ -0,0 +1,70 @@ +# Copyright 2016 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import datetime + +from oslo_config import cfg +#from oslo_db.sqlalchemy import session as db_session +import six +import sqlalchemy +from sqlalchemy.engine.url import make_url + +import logging + +from coverage2sql.db import models +#from coverage2sql import exceptions +#from coverage2sql import read_coverage + +CONF = cfg.CONF +CONF.register_cli_opt(cfg.BoolOpt('verbose', short='v', default=False, + help='Verbose output including logging of ' + 'SQL statements')) + +DAY_SECONDS = 60 * 60 * 24 + +_facades = {} + + +def _create_facade_lazily(): + global _facades + db_url = make_url(CONF.database.connection) + db_backend = db_url.get_backend_name() + facade = _facades.get(db_backend) + if facade is None: + facade = db_session.EngineFacade( + CONF.database.connection, + **dict(six.iteritems(CONF.database))) + _facades[db_backend] = facade + return facade + + +def get_session(autocommit=True, expire_on_commit=False): + """Get a new sqlalchemy Session instance + + :param bool autocommit: Enable autocommit mode for the session. + :param bool expire_on_commit: Expire the session on commit defaults False. + """ + facade = _create_facade_lazily() + session = facade.get_session(autocommit=autocommit, + expire_on_commit=expire_on_commit) + + # if --verbose was specified, turn on SQL logging + # note that this is done after the session has been initialized so that + # we can override the default sqlalchemy logging + if CONF.get('verbose', False): + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + return session diff --git a/coverage2sql/db/models.py b/coverage2sql/db/models.py new file mode 100644 index 0000000..7308774 --- /dev/null +++ b/coverage2sql/db/models.py @@ -0,0 +1,58 @@ +# Copyright 2016 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import uuid + +#from oslo_db.sqlalchemy import models # noqa +import six +import sqlalchemy as sa +from sqlalchemy.ext import declarative + +BASE = declarative.declarative_base() + + +class CoverageBase(object): + """Base class for Coverage Models.""" + __table_args__ = {'mysql_engine': 'InnoDB'} + __table_initialized__ = False + + def save(self, session=None): + from coverage2sql.db import api as db_api + super(CoverageBase, self).save(session or db_api.get_session()) + + def keys(self): + return list(self.__dict__.keys()) + + def values(self): + return self.__dict__.values() + + def items(self): + return self.__dict__.items() + + def to_dict(self): + d = self.__dict__.copy() + d.pop("_sa_instance_state") + return d + + +class Coverage(BASE, CoverageBase): + __tablename__ = 'coverages' + __table_args__ = (sa.Index('ix_project_name', 'project_name'), ) + id = sa.Column(sa.BigInteger, primary_key=True) + project_name = sa.Column(sa.String(256), + nullable=False) + coverage_rate = sa.Column(sa.Float()) + report_time = sa.Column(sa.DateTime()) + report_time_microsecond = sa.Column(sa.Integer(), default=0) diff --git a/coverage2sql/migrations/README b/coverage2sql/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/coverage2sql/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/coverage2sql/migrations/cli.py b/coverage2sql/migrations/cli.py new file mode 100644 index 0000000..f7500e5 --- /dev/null +++ b/coverage2sql/migrations/cli.py @@ -0,0 +1,186 @@ +# Copyright 2012 New Dream Network, LLC (DreamHost) +# Copyright 2016 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +from alembic import command as alembic_command +from alembic import config as alembic_config +from alembic import script as alembic_script +from alembic import util as alembic_util +from oslo_config import cfg +#from oslo_db import options + +from coverage2sql.db import api as db_api + +HEAD_FILENAME = 'HEAD' + + +def state_path_def(*args): + """Return an uninterpolated path relative to $state_path.""" + return os.path.join('$state_path', *args) + + +MIGRATION_OPTS = [ + cfg.BoolOpt('disable-microsecond-data-migration', short='d', default=False, + help="If set to true this option will skip the data migration" + " part of the microsecond migration. The schema changes " + "will still be run. If the database has already stripped " + "out the microseconds from the timestamps this will skip " + "converting the microsecond field from the timestamps " + "into a separate column"), +] + +CONF = cfg.CONF +#CONF.register_cli_opts(options.database_opts, group='database') +CONF.register_cli_opts(MIGRATION_OPTS) +CONF.import_opt('verbose', 'coverage2sql.db.api') + + +def do_alembic_command(config, cmd, *args, **kwargs): + try: + getattr(alembic_command, cmd)(config, *args, **kwargs) + except alembic_util.CommandError as e: + alembic_util.err(str(e)) + + +def do_check_migration(config, cmd): + do_alembic_command(config, 'branches') + validate_head_file(config) + + +def do_upgrade_downgrade(config, cmd): + if not CONF.command.revision and not CONF.command.delta: + raise SystemExit('You must provide a revision or relative delta') + + revision = CONF.command.revision + + if CONF.command.delta: + sign = '+' if CONF.command.name == 'upgrade' else '-' + revision = sign + str(CONF.command.delta) + else: + revision = CONF.command.revision + + do_alembic_command(config, cmd, revision, sql=CONF.command.sql) + + +def do_stamp(config, cmd): + do_alembic_command(config, cmd, + CONF.command.revision, + sql=CONF.command.sql) + + +def do_revision(config, cmd): + do_alembic_command(config, cmd, + message=CONF.command.message, + autogenerate=CONF.command.autogenerate, + sql=CONF.command.sql) + update_head_file(config) + + +def validate_head_file(config): + script = alembic_script.ScriptDirectory.from_config(config) + if len(script.get_heads()) > 1: + alembic_util.err('Timeline branches unable to generate timeline') + + head_path = os.path.join(script.versions, HEAD_FILENAME) + if (os.path.isfile(head_path) and + open(head_path).read().strip() == script.get_current_head()): + return + else: + alembic_util.err('HEAD file does not match migration timeline head') + + +def expire_old(config, cmd): + expire_age = int(CONF.command.expire_age) + if not CONF.command.no_runs: + print('Expiring old runs.') + db_api.delete_old_runs(expire_age) + if not CONF.command.no_test_runs: + print('Expiring old test_runs') + db_api.delete_old_test_runs(expire_age) + + +def update_head_file(config): + script = alembic_script.ScriptDirectory.from_config(config) + if len(script.get_heads()) > 1: + alembic_util.err('Timeline branches unable to generate timeline') + + head_path = os.path.join(script.versions, HEAD_FILENAME) + with open(head_path, 'w+') as f: + f.write(script.get_current_head()) + + +def add_command_parsers(subparsers): + for name in ['current', 'history', 'branches']: + parser = subparsers.add_parser(name) + parser.set_defaults(func=do_alembic_command) + + parser = subparsers.add_parser('check_migration') + parser.set_defaults(func=do_check_migration) + + for name in ['upgrade', 'downgrade']: + parser = subparsers.add_parser(name) + parser.add_argument('--delta', type=int) + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision', nargs='?') + parser.add_argument('--mysql-engine', + default='', + help='Change MySQL storage engine of current ' + 'existing tables') + parser.set_defaults(func=do_upgrade_downgrade) + + parser = subparsers.add_parser('stamp') + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision') + parser.set_defaults(func=do_stamp) + + parser = subparsers.add_parser('revision') + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.add_argument('--sql', action='store_true') + parser.set_defaults(func=do_revision) + + parser = subparsers.add_parser('expire', + help="delete old rows from runs and " + "test_runs tables") + parser.add_argument('--no-runs', action='store_true', + help="Don't delete any rows in the runs table") + parser.add_argument('--no-test-runs', action='store_true', + help="Don't delete any rows in the test_runs table") + parser.add_argument('--expire-age', '-e', default=186, + help="Number of days into the past to use as the " + "expiration point") + parser.set_defaults(func=expire_old) + +command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + +CONF.register_cli_opt(command_opt) + + +def main(): + config = alembic_config.Config(os.path.join(os.path.dirname(__file__), + 'alembic.ini')) + config.set_main_option('script_location', + 'coverage2sql:migrations') + config.coverage2sql_config = CONF + CONF() + CONF.command.func(config, CONF.command.name) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/coverage2sql/migrations/env.py b/coverage2sql/migrations/env.py new file mode 100644 index 0000000..2827d76 --- /dev/null +++ b/coverage2sql/migrations/env.py @@ -0,0 +1,85 @@ +# Copyright 2016 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/coverage2sql/migrations/script.py.mako b/coverage2sql/migrations/script.py.mako new file mode 100644 index 0000000..43c0940 --- /dev/null +++ b/coverage2sql/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py b/coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py new file mode 100644 index 0000000..c352d22 --- /dev/null +++ b/coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py @@ -0,0 +1,31 @@ +"""Add coverages table + +Revision ID: 52dfb338f74e +Revises: +Create Date: 2016-04-19 18:16:52.780046 + +""" + +# revision identifiers, used by Alembic. +revision = '52dfb338f74e' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table('coverages', + sa.Column('id', sa.BigInteger(), primary_key=True), + sa.Column('project_name', sa.String(256), nullable=False), + sa.Column('coverage_rate', sa.Float()), + sa.Column('report_time', sa.DateTime()), + sa.Column('report_time_microsecond', sa.Integer(), + default=0), + mysql_engine='InnoDB') + + +def downgrade(): + pass diff --git a/coverage2sql/shell.py b/coverage2sql/shell.py new file mode 100644 index 0000000..4ef219c --- /dev/null +++ b/coverage2sql/shell.py @@ -0,0 +1,76 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import sys + +from oslo_config import cfg +# from oslo_db import options +from pbr import version +from stevedore import enabled + +from coverage2sql.db import api +# from coverage2sql import exceptions +# from coverage2sql import read_subunit as subunit + +CONF = cfg.CONF +CONF.import_opt('verbose', 'coverage2sql.db.api') + +SHELL_OPTS = [ + cfg.MultiStrOpt('coverage_files', positional=True, + help='list of coverage files to put into the database'), +] + +_version_ = version.VersionInfo('coverage2sql').version_string() + + +def cli_opts(): + for opt in SHELL_OPTS: + CONF.register_cli_opt(opt) + + +def list_opts(): + """Return a list of oslo.config options available. + + The purpose of this is to allow tools like the Oslo sample config file + generator to discover the options exposed to users. + """ + return [('DEFAULT', copy.deepcopy(SHELL_OPTS))] + + +def parse_args(argv, default_config_files=None): + # cfg.CONF.register_cli_opts(options.database_opts, group='database') + cfg.CONF(argv[1:], project='coverage2sql', version=_version_, + default_config_files=default_config_files) + + +def process_results(results): + print(results) + + +def main(): + cli_opts() + + parse_args(sys.argv) + if CONF.coverage_files: + print("From file:") + process_results("FIXME") # FIXME + else: + print("From stdin:") + process_results("FIXME") # FIXME + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/requirements.txt b/requirements.txt index 30806d5..f0a3376 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,6 @@ # process, which may cause wedges in the gate later. pbr>=1.6 +SQLAlchemy>=0.8.2 +alembic>=0.4.1 +oslo.config>=1.4.0.0a3 diff --git a/setup.cfg b/setup.cfg index 8dfc3bb..586e69c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,13 +16,23 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 [files] packages = coverage2sql +[global] +setup-hooks = + pbr.hooks.setup_hook + +[entry_points] +console_scripts = + coverage2sql = coverage2sql.shell:main + coverage2sql-db-manage = coverage2sql.migrations.cli:main +oslo.config.opts = + coverage2sql.shell = coverage2sql.shell:list_opts + [build_sphinx] source-dir = doc/source build-dir = doc/build diff --git a/test-requirements.txt b/test-requirements.txt index 3858eb7..df1bc12 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,6 +8,7 @@ coverage>=3.6 python-subunit>=0.0.18 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 oslosphinx>=2.5.0 # Apache-2.0 +PyMySql oslotest>=1.10.0 # Apache-2.0 testrepository>=0.0.18 testscenarios>=0.4