Add migration and shell

.gitignore

@@ -1,3 +1,5 @@
+alembic.ini
+
 *.py[cod]
 
 # C extensions

coverage2sql/db/api.py

@@ -0,0 +1,70 @@
+# Copyright 2016 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import collections
+import datetime
+
+from oslo_config import cfg
+#from oslo_db.sqlalchemy import session as db_session
+import six
+import sqlalchemy
+from sqlalchemy.engine.url import make_url
+
+import logging
+
+from coverage2sql.db import models
+#from coverage2sql import exceptions
+#from coverage2sql import read_coverage
+
+CONF = cfg.CONF
+CONF.register_cli_opt(cfg.BoolOpt('verbose', short='v', default=False,
+                                  help='Verbose output including logging of '
+                                       'SQL statements'))
+
+DAY_SECONDS = 60 * 60 * 24
+
+_facades = {}
+
+
+def _create_facade_lazily():
+    global _facades
+    db_url = make_url(CONF.database.connection)
+    db_backend = db_url.get_backend_name()
+    facade = _facades.get(db_backend)
+    if facade is None:
+        facade = db_session.EngineFacade(
+            CONF.database.connection,
+            **dict(six.iteritems(CONF.database)))
+        _facades[db_backend] = facade
+    return facade
+
+
+def get_session(autocommit=True, expire_on_commit=False):
+    """Get a new sqlalchemy Session instance
+
+    :param bool autocommit: Enable autocommit mode for the session.
+    :param bool expire_on_commit: Expire the session on commit defaults False.
+    """
+    facade = _create_facade_lazily()
+    session = facade.get_session(autocommit=autocommit,
+                                 expire_on_commit=expire_on_commit)
+
+    # if --verbose was specified, turn on SQL logging
+    # note that this is done after the session has been initialized so that
+    # we can override the default sqlalchemy logging
+    if CONF.get('verbose', False):
+        logging.basicConfig()
+        logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+
+    return session

coverage2sql/db/models.py

@@ -0,0 +1,58 @@
+# Copyright 2016 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import datetime
+import uuid
+
+#from oslo_db.sqlalchemy import models  # noqa
+import six
+import sqlalchemy as sa
+from sqlalchemy.ext import declarative
+
+BASE = declarative.declarative_base()
+
+
+class CoverageBase(object):
+    """Base class for Coverage Models."""
+    __table_args__ = {'mysql_engine': 'InnoDB'}
+    __table_initialized__ = False
+
+    def save(self, session=None):
+        from coverage2sql.db import api as db_api
+        super(CoverageBase, self).save(session or db_api.get_session())
+
+    def keys(self):
+        return list(self.__dict__.keys())
+
+    def values(self):
+        return self.__dict__.values()
+
+    def items(self):
+        return self.__dict__.items()
+
+    def to_dict(self):
+        d = self.__dict__.copy()
+        d.pop("_sa_instance_state")
+        return d
+
+
+class Coverage(BASE, CoverageBase):
+    __tablename__ = 'coverages'
+    __table_args__ = (sa.Index('ix_project_name', 'project_name'), )
+    id = sa.Column(sa.BigInteger, primary_key=True)
+    project_name = sa.Column(sa.String(256),
+                             nullable=False)
+    coverage_rate = sa.Column(sa.Float())
+    report_time = sa.Column(sa.DateTime())
+    report_time_microsecond = sa.Column(sa.Integer(), default=0)

coverage2sql/migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.

coverage2sql/migrations/cli.py

@@ -0,0 +1,186 @@
+# Copyright 2012 New Dream Network, LLC (DreamHost)
+# Copyright 2016 Hewlett Packard Enterprise Development LP
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import sys
+
+from alembic import command as alembic_command
+from alembic import config as alembic_config
+from alembic import script as alembic_script
+from alembic import util as alembic_util
+from oslo_config import cfg
+#from oslo_db import options
+
+from coverage2sql.db import api as db_api
+
+HEAD_FILENAME = 'HEAD'
+
+
+def state_path_def(*args):
+    """Return an uninterpolated path relative to $state_path."""
+    return os.path.join('$state_path', *args)
+
+
+MIGRATION_OPTS = [
+    cfg.BoolOpt('disable-microsecond-data-migration', short='d', default=False,
+                help="If set to true this option will skip the data migration"
+                     " part of the microsecond migration. The schema changes "
+                     "will still be run. If the database has already stripped "
+                     "out the microseconds from the timestamps this will skip "
+                     "converting the microsecond field from the timestamps "
+                     "into a separate column"),
+]
+
+CONF = cfg.CONF
+#CONF.register_cli_opts(options.database_opts, group='database')
+CONF.register_cli_opts(MIGRATION_OPTS)
+CONF.import_opt('verbose', 'coverage2sql.db.api')
+
+
+def do_alembic_command(config, cmd, *args, **kwargs):
+    try:
+        getattr(alembic_command, cmd)(config, *args, **kwargs)
+    except alembic_util.CommandError as e:
+        alembic_util.err(str(e))
+
+
+def do_check_migration(config, cmd):
+    do_alembic_command(config, 'branches')
+    validate_head_file(config)
+
+
+def do_upgrade_downgrade(config, cmd):
+    if not CONF.command.revision and not CONF.command.delta:
+        raise SystemExit('You must provide a revision or relative delta')
+
+    revision = CONF.command.revision
+
+    if CONF.command.delta:
+        sign = '+' if CONF.command.name == 'upgrade' else '-'
+        revision = sign + str(CONF.command.delta)
+    else:
+        revision = CONF.command.revision
+
+    do_alembic_command(config, cmd, revision, sql=CONF.command.sql)
+
+
+def do_stamp(config, cmd):
+    do_alembic_command(config, cmd,
+                       CONF.command.revision,
+                       sql=CONF.command.sql)
+
+
+def do_revision(config, cmd):
+    do_alembic_command(config, cmd,
+                       message=CONF.command.message,
+                       autogenerate=CONF.command.autogenerate,
+                       sql=CONF.command.sql)
+    update_head_file(config)
+
+
+def validate_head_file(config):
+    script = alembic_script.ScriptDirectory.from_config(config)
+    if len(script.get_heads()) > 1:
+        alembic_util.err('Timeline branches unable to generate timeline')
+
+    head_path = os.path.join(script.versions, HEAD_FILENAME)
+    if (os.path.isfile(head_path) and
+        open(head_path).read().strip() == script.get_current_head()):
+        return
+    else:
+        alembic_util.err('HEAD file does not match migration timeline head')
+
+
+def expire_old(config, cmd):
+    expire_age = int(CONF.command.expire_age)
+    if not CONF.command.no_runs:
+        print('Expiring old runs.')
+        db_api.delete_old_runs(expire_age)
+    if not CONF.command.no_test_runs:
+        print('Expiring old test_runs')
+        db_api.delete_old_test_runs(expire_age)
+
+
+def update_head_file(config):
+    script = alembic_script.ScriptDirectory.from_config(config)
+    if len(script.get_heads()) > 1:
+        alembic_util.err('Timeline branches unable to generate timeline')
+
+    head_path = os.path.join(script.versions, HEAD_FILENAME)
+    with open(head_path, 'w+') as f:
+        f.write(script.get_current_head())
+
+
+def add_command_parsers(subparsers):
+    for name in ['current', 'history', 'branches']:
+        parser = subparsers.add_parser(name)
+        parser.set_defaults(func=do_alembic_command)
+
+    parser = subparsers.add_parser('check_migration')
+    parser.set_defaults(func=do_check_migration)
+
+    for name in ['upgrade', 'downgrade']:
+        parser = subparsers.add_parser(name)
+        parser.add_argument('--delta', type=int)
+        parser.add_argument('--sql', action='store_true')
+        parser.add_argument('revision', nargs='?')
+        parser.add_argument('--mysql-engine',
+                            default='',
+                            help='Change MySQL storage engine of current '
+                                 'existing tables')
+        parser.set_defaults(func=do_upgrade_downgrade)
+
+    parser = subparsers.add_parser('stamp')
+    parser.add_argument('--sql', action='store_true')
+    parser.add_argument('revision')
+    parser.set_defaults(func=do_stamp)
+
+    parser = subparsers.add_parser('revision')
+    parser.add_argument('-m', '--message')
+    parser.add_argument('--autogenerate', action='store_true')
+    parser.add_argument('--sql', action='store_true')
+    parser.set_defaults(func=do_revision)
+
+    parser = subparsers.add_parser('expire',
+                                   help="delete old rows from runs and "
+                                        "test_runs tables")
+    parser.add_argument('--no-runs', action='store_true',
+                        help="Don't delete any rows in the runs table")
+    parser.add_argument('--no-test-runs', action='store_true',
+                        help="Don't delete any rows in the test_runs table")
+    parser.add_argument('--expire-age', '-e', default=186,
+                        help="Number of days into the past to use as the "
+                             "expiration point")
+    parser.set_defaults(func=expire_old)
+
+command_opt = cfg.SubCommandOpt('command',
+                                title='Command',
+                                help='Available commands',
+                                handler=add_command_parsers)
+
+CONF.register_cli_opt(command_opt)
+
+
+def main():
+    config = alembic_config.Config(os.path.join(os.path.dirname(__file__),
+                                                'alembic.ini'))
+    config.set_main_option('script_location',
+                           'coverage2sql:migrations')
+    config.coverage2sql_config = CONF
+    CONF()
+    CONF.command.func(config, CONF.command.name)
+
+if __name__ == "__main__":
+    sys.exit(main())

coverage2sql/migrations/env.py

@@ -0,0 +1,85 @@
+# Copyright 2016 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

coverage2sql/migrations/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py

@@ -0,0 +1,31 @@
+"""Add coverages table
+
+Revision ID: 52dfb338f74e
+Revises:
+Create Date: 2016-04-19 18:16:52.780046
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '52dfb338f74e'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table('coverages',
+                    sa.Column('id', sa.BigInteger(), primary_key=True),
+                    sa.Column('project_name', sa.String(256), nullable=False),
+                    sa.Column('coverage_rate', sa.Float()),
+                    sa.Column('report_time', sa.DateTime()),
+                    sa.Column('report_time_microsecond', sa.Integer(),
+                              default=0),
+                    mysql_engine='InnoDB')
+
+
+def downgrade():
+    pass

coverage2sql/shell.py

@@ -0,0 +1,76 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import sys
+
+from oslo_config import cfg
+# from oslo_db import options
+from pbr import version
+from stevedore import enabled
+
+from coverage2sql.db import api
+# from coverage2sql import exceptions
+# from coverage2sql import read_subunit as subunit
+
+CONF = cfg.CONF
+CONF.import_opt('verbose', 'coverage2sql.db.api')
+
+SHELL_OPTS = [
+    cfg.MultiStrOpt('coverage_files', positional=True,
+                    help='list of coverage files to put into the database'),
+]
+
+_version_ = version.VersionInfo('coverage2sql').version_string()
+
+
+def cli_opts():
+    for opt in SHELL_OPTS:
+        CONF.register_cli_opt(opt)
+
+
+def list_opts():
+    """Return a list of oslo.config options available.
+
+    The purpose of this is to allow tools like the Oslo sample config file
+    generator to discover the options exposed to users.
+    """
+    return [('DEFAULT', copy.deepcopy(SHELL_OPTS))]
+
+
+def parse_args(argv, default_config_files=None):
+    # cfg.CONF.register_cli_opts(options.database_opts, group='database')
+    cfg.CONF(argv[1:], project='coverage2sql', version=_version_,
+             default_config_files=default_config_files)
+
+
+def process_results(results):
+    print(results)
+
+
+def main():
+    cli_opts()
+
+    parse_args(sys.argv)
+    if CONF.coverage_files:
+        print("From file:")
+        process_results("FIXME")  # FIXME
+    else:
+        print("From stdin:")
+        process_results("FIXME")  # FIXME
+
+
+if __name__ == "__main__":
+    sys.exit(main())

requirements.txt

@@ -3,3 +3,6 @@
 # process, which may cause wedges in the gate later.
 
 pbr>=1.6
+SQLAlchemy>=0.8.2
+alembic>=0.4.1
+oslo.config>=1.4.0.0a3

setup.cfg

@@ -16,13 +16,23 @@ classifier =
     Programming Language :: Python :: 2
     Programming Language :: Python :: 2.7
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.3
     Programming Language :: Python :: 3.4
 
 [files]
 packages =
     coverage2sql
 
+[global]
+setup-hooks =
+    pbr.hooks.setup_hook
+
+[entry_points]
+console_scripts =
+    coverage2sql = coverage2sql.shell:main
+    coverage2sql-db-manage = coverage2sql.migrations.cli:main
+oslo.config.opts =
+    coverage2sql.shell = coverage2sql.shell:list_opts
+
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build

test-requirements.txt

@@ -8,6 +8,7 @@ coverage>=3.6
 python-subunit>=0.0.18
 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
 oslosphinx>=2.5.0 # Apache-2.0
+PyMySql
 oslotest>=1.10.0 # Apache-2.0
 testrepository>=0.0.18
 testscenarios>=0.4