From 9d86270fac0f9e98f21eaa896ce13a6192e17d89 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Sun, 8 Jun 2014 15:43:06 -0400 Subject: [PATCH] Initial commit This just adds the basic framework for all the various pieces. The schema will be built using alembic. Everything else is untested. --- .gitignore | 16 ++ LICENSE | 176 ++++++++++++++++++ README.rst | 20 ++ alembic.ini | 59 ++++++ openstack-common.conf | 11 ++ requirements.txt | 6 + setup.cfg | 37 ++++ setup.py | 29 +++ subunit2sql/__init__.py | 0 subunit2sql/db/__init__.py | 0 subunit2sql/db/api.py | 159 ++++++++++++++++ subunit2sql/db/models.py | 80 ++++++++ subunit2sql/exceptions.py | 46 +++++ subunit2sql/migrations/README | 1 + subunit2sql/migrations/env.py | 83 +++++++++ subunit2sql/migrations/script.py.mako | 22 +++ .../1f92cfe8a6d3_create_runs_table.py | 29 +++ .../3db7b49816d5_create_test_runs_table.py | 33 ++++ .../5ef013efbc2_create_tests_tables.py | 28 +++ subunit2sql/shell.py | 59 ++++++ subunit2sql/subunit.py | 23 +++ subunit2sql/tests/__init__.py | 0 subunit2sql/tests/base.py | 13 ++ subunit2sql/tests/db/__init__.py | 0 subunit2sql/tests/migrations/__init__.py | 0 subunit2sql/tests/subunit.py | 15 ++ test-requirements.txt | 9 + tox.ini | 36 ++++ 28 files changed, 990 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.rst create mode 100644 alembic.ini create mode 100644 openstack-common.conf create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 subunit2sql/__init__.py create mode 100644 subunit2sql/db/__init__.py create mode 100644 subunit2sql/db/api.py create mode 100644 subunit2sql/db/models.py create mode 100644 subunit2sql/exceptions.py create mode 100644 subunit2sql/migrations/README create mode 100644 subunit2sql/migrations/env.py create mode 100644 subunit2sql/migrations/script.py.mako create mode 100644 subunit2sql/migrations/versions/1f92cfe8a6d3_create_runs_table.py create mode 100644 subunit2sql/migrations/versions/3db7b49816d5_create_test_runs_table.py create mode 100644 subunit2sql/migrations/versions/5ef013efbc2_create_tests_tables.py create mode 100644 subunit2sql/shell.py create mode 100644 subunit2sql/subunit.py create mode 100644 subunit2sql/tests/__init__.py create mode 100644 subunit2sql/tests/base.py create mode 100644 subunit2sql/tests/db/__init__.py create mode 100644 subunit2sql/tests/migrations/__init__.py create mode 100644 subunit2sql/tests/subunit.py create mode 100644 test-requirements.txt create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cfc631a --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +AUTHORS +ChangeLog +*.pyc +*.log +*.swp +*.swo +*.egg* +.tox +.venv +*.db +dist +build +.testrepository +.coverage* +!.coveragerc +cover/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..68c771a --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..058190b --- /dev/null +++ b/README.rst @@ -0,0 +1,20 @@ +subunit2SQL README +================== + +subunit2SQL like it's name implies is a tool used for converting subunit +streams to data in a SQL database. The motivation is that for multiple +distributed test runs that are generating subunit output it is useful to +store the results in a unified repository. This is the motivation for the +testrepository project which does a good job for centralizing the results from +multiple test runs. + +Imagine something like the OpenStack CI system where the same basic test suite +is normally run several hundreds of times a day. To provide useful +introspection on the data from those runs and to build trends over time +the test results need to be stored in a format that allows for easy querying. +SQL databases make a lot of sense for doing this. + +subunit2SQL uses alembic migrations to setup a DB schema that can then be used +by the subunit2sql binary to parse subunit streams and populate the DB. +Additional it provides a DB API that can be used to query information from the +results stored to build other tooling. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..a607501 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,59 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = subunit2sql/migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +sqlalchemy.url = sqlite:///test.db + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/openstack-common.conf b/openstack-common.conf new file mode 100644 index 0000000..0063b5f --- /dev/null +++ b/openstack-common.conf @@ -0,0 +1,11 @@ +[DEFAULT] + +# The list of modules to copy from openstack-common +module=config +module=install_venv_common +module=log +module=importlib +module=fixture + +# The base module to hold the copy of openstack.common +base=subunit2sql diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1d5583f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +alembic>=0.4.1 +oslo.config>=1.2.0 +oslo.db +pbr>=0.6,<1.0 +six>=1.5.2 +SQLAlchemy>=0.7.8,<=0.8.99 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..9f6e2a4 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,37 @@ +[metadata] +name = subunit2sql +summary = Command to Read a subunit file or stream and put the data in SQL DB +description-file = + README.rst +license = Apache License, Version 2.0 +author = Matthew Treinish +author-email = openstack-dev@lists.openstack.org +classifier = + Development Status :: 5 - Production/Stable + Environment :: Console + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.3 + +[files] +packages = + subunit2sql + +[entry_points] +console_scripts = + subunit2sql = subunit2sql.shell:main + +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 + +[upload_sphinx] +upload-dir = doc/build/html + +[wheel] +universal = 1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ee06f22 --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/subunit2sql/__init__.py b/subunit2sql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/subunit2sql/db/__init__.py b/subunit2sql/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/subunit2sql/db/api.py b/subunit2sql/db/api.py new file mode 100644 index 0000000..9fec77a --- /dev/null +++ b/subunit2sql/db/api.py @@ -0,0 +1,159 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg +from oslo.db.sqlalchemy import session as db_session +from oslo.db.sqlalchemy import utils as db_utils + +from subunit2sql.db import models +from subunit2sql import exceptions + +CONF = cfg.CONF + +DAY_SECONDS = 60 * 60 * 24 + +_FACADE = None + + +def _create_facade_lazily(): + global _FACADE + if _FACADE is None: + _FACADE = db_session.EngineFacade( + CONF.database.connection, + **dict(CONF.database.iteritems())) + return _FACADE + + +def get_session(autocommit=True, expire_on_commit=False): + facade = _create_facade_lazily() + return facade.get_session(autocommit=autocommit, + expire_on_commit=expire_on_commit) + + +def create_test(test_id, run_count=0, success=0, failure=0): + """Create a new test record in the database + + :param test_id: test_id identifying the test + :param run_count: total number or runs + :param success: number of successful runs + :param failure: number of failed runs + + Raises InvalidRunCount if the run_count doesn't equal the sum of the + successes and failures. + """ + if run_count != success + failure: + raise exceptions.InvalidRunCount() + test = models.Test() + test.test_id = test_id + test.run_count = run_count + test.success = success + test.failure = failure + session = get_session() + with session.begin(): + session.add(test) + return test + + +def create_run(skips=0, fails=0, passes=0, run_time=0, artifacts=None): + """Create a new run record in the database + + :param skips: total number of skiped tests + :param fails: total number of failed tests + :param passes: total number of passed tests + :param run_time: total run time + :param artifacts: A link to any artifacts from the test run + """ + run = models.Run() + run.skips = skips + run.fails = fails + run.passes = passes + run.run_time = run_time + if artifacts: + run.artifacts = artifacts + session = get_session() + with session.begin(): + session.add(run) + return run + + +def create_test_run(test_id, run_id, status, start_time=None, + end_time=None): + """Create a new test run record in the database + + :param test_id: uuid for test that was run + :param run_id: uuid for run that this was a member of + :param start_time: when the test was started + :param end_time: when the test was finished + """ + test_run = models.TestRun() + test_run.test_id = test_id + test_run.run_id = run_id + test_run.end_time = end_time + test_run.start_time = start_time + session = get_session() + with session.begin(): + session.add(test_run) + return test_run + + +def get_all_tests(): + query = db_utils.model_query(models.Test) + return query.all() + + +def get_all_runs(): + query = db_utils.models_query(models.Run) + return query.all() + + +def get_all_test_runs(test_id): + query = db_utils.models_query(models.TestRun) + return query.all() + + +def get_test_run_by_id(test_run_id, session=None): + session = session or get_session() + test_run = db_utils.model_query(models.TestRun, session=session).filter_by( + id=test_run_id).first() + return test_run + + +def get_test_runs_by_test_id(test_id, session=None): + session = session or get_session() + test_runs = db_utils.model_query(models.TestRun, + session=session).filter_by( + test_id=test_id).all() + return test_runs + + +def get_test_runs_by_run_id(run_id, session=None): + session = session or get_session() + test_runs = db_utils.model_query(models.Run, session=session).filter_by( + run_id=run_id).all() + return test_runs + + +def get_test_run_duration(test_run_id): + session = get_session() + + test_run = get_test_run_by_id(test_run_id, session) + start = test_run.start_time + end = test_run.end_time + if not start or not end: + duration = '' + else: + delta = end - start + duration = '%d.%06ds' % ( + delta.days * DAY_SECONDS + delta.seconds, delta.microseconds) + return duration diff --git a/subunit2sql/db/models.py b/subunit2sql/db/models.py new file mode 100644 index 0000000..88f9dd3 --- /dev/null +++ b/subunit2sql/db/models.py @@ -0,0 +1,80 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid + +from oslo.db import models # noqa +import sqlalchemy as sa + + +class SubunitBase(models.ModelBase, models.TimeStampMixin): + """Base class for Subunit Models.""" + __table_args__ = {'mysql_engine': 'InnoDB'} + __table_initialized__ = False + + def save(self, session=None): + from subunit2sql.db import api as db_api + super(SubunitBase, self).save(session or db_api.get_session()) + + def keys(self): + return self.__dict__.keys() + + def values(self): + return self.__dict__.values() + + def items(self): + return self.__dict__.items() + + def to_dict(self): + return self.__dict__.copy() + + +class Test(SubunitBase): + __tablename__ = 'tests' + __table_args__ = () + id = sa.Column(sa.String(36), primary_key=True, + default=lambda: str(uuid.uuid4())) + test_id = sa.String(256) + run_count = sa.Integer() + success = sa.Integer() + failure = sa.Integer() + + +class Run(SubunitBase): + __tablename__ = 'runs' + __table_args__ = () + id = sa.Column(sa.String(36), primary_key=True, + default=lambda: str(uuid.uuid4())) + skips = sa.Integer() + fails = sa.Integer() + passes = sa.Integer() + run_time = sa.Integer() + artifacts = sa.Text() + + +class TestRun(SubunitBase): + __tablename__ = 'test_run' + __table_args__ = (sa.Index('ix_test_run_test_id', 'test_id'), + sa.Index('ix_test_run_run_id', 'run_id'), + sa.UniqueConstraint('test_id', 'run_id', + name='ix_test_run_test_id_run_id')) + + id = sa.Column(sa.String(36), primary_key=True, + default=lambda: str(uuid.uuid4())) + test_id = sa.Column(sa.String(36), sa.ForeignKey('tests.id'), + nullable=False) + run_id = sa.Column(sa.String(36), sa.ForeignKey('runs.id'), nullable=False) + status = sa.Column(sa.String(256)) + start_time = sa.DateTime() + end_time = sa.DateTime() diff --git a/subunit2sql/exceptions.py b/subunit2sql/exceptions.py new file mode 100644 index 0000000..8456696 --- /dev/null +++ b/subunit2sql/exceptions.py @@ -0,0 +1,46 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Subunit2SQLException(Exception): + """Base Subunit2SQL Exception. + + To correctly use this class, inherit from it and define + a 'message' property. That message will get printf'd + with the keyword arguments provided to the constructor. + """ + message = "An unknown exception occurred" + + def __init__(self, *args, **kwargs): + super(Subunit2SQLException, self).__init__() + try: + self._error_string = self.message % kwargs + except Exception: + # at least get the core message out if something happened + self._error_string = self.message + if len(args) > 0: + # If there is a non-kwarg parameter, assume it's the error + # message or reason description and tack it on to the end + # of the exception message + # Convert all arguments into their string representations... + args = ["%s" % arg for arg in args] + self._error_string = (self._error_string + + "\nDetails: %s" % '\n'.join(args)) + + def __str__(self): + return self._error_string + + +class InvalidRunCount(Subunit2SQLException): + message = "Invalid Run Count" diff --git a/subunit2sql/migrations/README b/subunit2sql/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/subunit2sql/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/subunit2sql/migrations/env.py b/subunit2sql/migrations/env.py new file mode 100644 index 0000000..984c42d --- /dev/null +++ b/subunit2sql/migrations/env.py @@ -0,0 +1,83 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig # noqa + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = engine_from_config(config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure(connection=connection, target_metadata=target_metadata) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/subunit2sql/migrations/script.py.mako b/subunit2sql/migrations/script.py.mako new file mode 100644 index 0000000..9570201 --- /dev/null +++ b/subunit2sql/migrations/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/subunit2sql/migrations/versions/1f92cfe8a6d3_create_runs_table.py b/subunit2sql/migrations/versions/1f92cfe8a6d3_create_runs_table.py new file mode 100644 index 0000000..b1c8f67 --- /dev/null +++ b/subunit2sql/migrations/versions/1f92cfe8a6d3_create_runs_table.py @@ -0,0 +1,29 @@ +"""create runs table + +Revision ID: 1f92cfe8a6d3 +Revises: 5ef013efbc2 +Create Date: 2014-06-08 14:29:17.622700 + +""" + +# revision identifiers, used by Alembic. +revision = '1f92cfe8a6d3' +down_revision = '5ef013efbc2' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table('runs', + sa.Column('id', sa.String(36), primary_key=True), + sa.Column('skips', sa.Integer()), + sa.Column('fails', sa.Integer()), + sa.Column('pass', sa.Integer()), + sa.Column('run_time', sa.Integer()), + sa.Column('artifacts', sa.Text()), + mysql_engine=True) + + +def downgrade(): + op.drop_table('runs') diff --git a/subunit2sql/migrations/versions/3db7b49816d5_create_test_runs_table.py b/subunit2sql/migrations/versions/3db7b49816d5_create_test_runs_table.py new file mode 100644 index 0000000..b51b82e --- /dev/null +++ b/subunit2sql/migrations/versions/3db7b49816d5_create_test_runs_table.py @@ -0,0 +1,33 @@ +"""create test_runs table + +Revision ID: 3db7b49816d5 +Revises: 1f92cfe8a6d3 +Create Date: 2014-06-08 14:34:56.786781 + +""" + +# revision identifiers, used by Alembic. +revision = '3db7b49816d5' +down_revision = '1f92cfe8a6d3' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table('test_runs', + sa.Column('id', sa.String(36), primary_key=True), + sa.Column('test_id', sa.String(36), + sa.ForeignKey('tests.id'), + nullable=False, index=True), + sa.Column('run_id', sa.String(36), + sa.ForeignKey('runs.id'), + nullable=False, index=True), + sa.Column('status', sa.String(256)), + sa.Column('start_time', sa.DateTime()), + sa.Column('stop_time', sa.DateTime()), + mysql_engine='InnoDB') + + +def downgrade(): + op.drop_table('test_runs') diff --git a/subunit2sql/migrations/versions/5ef013efbc2_create_tests_tables.py b/subunit2sql/migrations/versions/5ef013efbc2_create_tests_tables.py new file mode 100644 index 0000000..360bda7 --- /dev/null +++ b/subunit2sql/migrations/versions/5ef013efbc2_create_tests_tables.py @@ -0,0 +1,28 @@ +"""create tests tables + +Revision ID: 5ef013efbc2 +Revises: None +Create Date: 2014-06-08 11:18:41.529268 + +""" + +# revision identifiers, used by Alembic. +revision = '5ef013efbc2' +down_revision = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table('tests', + sa.Column('id', sa.String(36), primary_key=True), + sa.Column('test_id', sa.String(256)), + sa.Column('run_count', sa.Integer()), + sa.Column('success', sa.Integer()), + sa.Column('failure', sa.Integer()), + mysql_engine='InnoDB') + + +def downgrade(): + op.drop_table('tables') diff --git a/subunit2sql/shell.py b/subunit2sql/shell.py new file mode 100644 index 0000000..0f7fae6 --- /dev/null +++ b/subunit2sql/shell.py @@ -0,0 +1,59 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from oslo.config import cfg +from oslo.db import options + +from subunit2sql import subunit + +shell_opts = [ + cfg.StrOpt('state_path', default='$pybasedir', + help='Top level dir for maintaining subunit2sql state'), + cfg.MultiStrOpt('subunit_files', positional=True) +] + +CONF = cfg.CONF +for opt in shell_opts: + CONF.register_cli_opt(opt) + + +def state_path_def(*args): + """Return an uninterpolated path relative to $state_path.""" + return os.path.join('$state_path', *args) + + +_DEFAULT_SQL_CONNECTION = 'sqlite:///' + state_path_def('subunit2sql.sqlite') + + +def parse_args(argv, default_config_files=None): + options.set_defaults(CONF, connection=_DEFAULT_SQL_CONNECTION, + sqlite_db='subunit2sql.sqlite') + CONF.register_opts(options.database_opts) + cfg.CONF(argv[1:], project='subunit2sql', + default_config_files=default_config_files) + +def main(): + parse_args(sys.argv) + if CONF.subunit_files: + streams = [ subunit.ReadSubunit(s) for s in subunit_files ] + else: + steams = [ subunit.ReadSubunit(sys.stdin) ] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/subunit2sql/subunit.py b/subunit2sql/subunit.py new file mode 100644 index 0000000..bdfec24 --- /dev/null +++ b/subunit2sql/subunit.py @@ -0,0 +1,23 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import subunit + +DAY_SECONDS = 60 * 60 * 24 + + +class ReadSubunit(object): + + def __init__(self, stream): + self.stream = subunit.ByteStreamToStreamResult(stream) diff --git a/subunit2sql/tests/__init__.py b/subunit2sql/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/subunit2sql/tests/base.py b/subunit2sql/tests/base.py new file mode 100644 index 0000000..f23179e --- /dev/null +++ b/subunit2sql/tests/base.py @@ -0,0 +1,13 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/subunit2sql/tests/db/__init__.py b/subunit2sql/tests/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/subunit2sql/tests/migrations/__init__.py b/subunit2sql/tests/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/subunit2sql/tests/subunit.py b/subunit2sql/tests/subunit.py new file mode 100644 index 0000000..989fd20 --- /dev/null +++ b/subunit2sql/tests/subunit.py @@ -0,0 +1,15 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..f22d5b1 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,9 @@ +hacking>=0.9.1,<0.10 +coverage>=3.6 +discover +fixtures>=0.3.14 +mock>=1.0 +sphinx>=1.1.2,<1.2 +oslosphinx +testrepository>=0.0.18 +testtools>=0.9.34 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..f826fb7 --- /dev/null +++ b/tox.ini @@ -0,0 +1,36 @@ +[tox] +minversion = 1.6 +envlist = py26,py27,py33,pep8 +skipsdist = True + +[testenv] +usedevelop = True +install_command = pip install -U --force-reinstall {opts} {packages} +setenv = VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + python setup.py test --slowest --testr-args='{posargs}' + +[testenv:pep8] +sitepackages = False +commands = + flake8 {posargs} + +[testenv:cover] +setenv = VIRTUAL_ENV={envdir} +commands = + python setup.py testr --coverage --testr-args='{posargs}' + +[testenv:venv] +commands = {posargs} + +[testenv:docs] +commands = python setup.py build_sphinx + + +[flake8] +# E125 is deliberately excluded. See https://github.com/jcrocholl/pep8/issues/126 + +ignore = E125 +exclude = .venv,.git,.tox,dist,doc,*egg,build