Add files table

This commit adds 'files' table. This files table has coverage_id
column as a foreign key for the coverages table. And this commit also
adds a release note for it.

Change-Id: I8998a79a1ba79bbdab1cd79810cc85bcbccbe7d8

coverage2sql/db/api.py

@@ -70,8 +70,8 @@ def get_session(autocommit=True, expire_on_commit=False):
     return session
 
 
-def create_coverage(project_name, coverage_rate=0.0, report_time=None,
-                    test_type='py27', session=None):
+def create_coverage(project_name, coverage_rate=0.0, rates=[],
+                    report_time=None, test_type='py27', session=None):
     """Create a new coverage record in the database.
 
     This method is used to add a new coverage in the database.
@@ -98,12 +98,41 @@ def create_coverage(project_name, coverage_rate=0.0, report_time=None,
         report_time_microsecond = None
     coverage.report_time = report_time
     coverage.report_time_microsecond = report_time_microsecond
+
     session = session or get_session()
     with session.begin():
         session.add(coverage)
+
     return coverage
 
 
+def add_file_rates(coverage_id, rates=[], session=None):
+    """Add rates a specific coverage.
+
+    This method is used to add rate records in the database.
+    It tracks the coverage history by individual files.
+
+    :param int coverage_id: coverage_id
+    :param list rates: rates dict list which has
+                       e.g. [{'filename': 'foo', 'line-rate': '0.1'}, ...]
+    :param session: optional session object if one isn't provided a new session
+                    will be acquired for the duration of this operation
+    :return: The list of created files objects
+    :rtype: coverage2sql.models.File
+    """
+    session = session or get_session()
+    files = []
+    with session.begin():
+        for r in rates:
+            f = models.File()
+            f.coverage_id = coverage_id
+            f.filename = r['filename']
+            f.line_rate = r['line-rate']
+            session.add(f)
+            files.append(f)
+    return files
+
+
 def get_coverage(project_name=None, test_type=None, session=None):
     """Get new coverage records in the database.
 

coverage2sql/db/models.py

@@ -54,3 +54,17 @@ class Coverage(BASE, CoverageBase):
     test_type = sa.Column(sa.String(256), nullable=False, default='py27')
     report_time = sa.Column(sa.DateTime(), default=datetime.datetime.utcnow())
     report_time_microsecond = sa.Column(sa.Integer(), default=0)
+
+
+class File(BASE, CoverageBase):
+    __tablename__ = 'files'
+    __table_args__ = (sa.Index('ix_file_coverage_id', 'coverage_id'),
+                      sa.Index('ix_filename', 'filename'))
+    id = sa.Column(sa.BigInteger, primary_key=True)
+    coverage_id = sa.Column(sa.BigInteger, nullable=False)
+    filename = sa.Column(sa.String(256), nullable=False)
+    line_rate = sa.Column(sa.Float())
+    coverage = sa.orm.relationship(Coverage,
+                                   backref=sa.orm.backref('file_coverage'),
+                                   foreign_keys=coverage_id,
+                                   primaryjoin=coverage_id == Coverage.id)

coverage2sql/migrations/versions/52dfb338f74e_add_coverages_table.py

@@ -52,4 +52,4 @@ def upgrade():
 
 
 def downgrade():
-    raise NotImplementedError()
+    op.drop_table('classes')

coverage2sql/migrations/versions/79dead6f7c26_add_files_table.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Add files table
+
+Revision ID: 79dead6f7c26
+Revises: cb0e61ce633e
+Create Date: 2017-02-07 17:57:28.777311
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '79dead6f7c26'
+down_revision = 'cb0e61ce633e'
+branch_labels = None
+depends_on = None
+
+from alembic import context
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    migration_context = context.get_context()
+    if migration_context.dialect.name == 'sqlite':
+        id_type = sa.Integer
+    else:
+        id_type = sa.BigInteger
+
+    op.create_table('files',
+                    sa.Column('id', id_type, autoincrement=True,
+                              primary_key=True),
+                    sa.Column('coverage_id', id_type, nullable=False),
+                    sa.Column('filename', sa.String(256), nullable=False),
+                    sa.Column('line_rate', sa.Float()),
+                    mysql_engine='InnoDB')
+    op.create_index('ix_class_coverage_id', 'files', ['coverage_id'])
+    op.create_index('ix_filename', 'files', ['filename'])
+
+
+def downgrade():
+    op.drop_table('files')

coverage2sql/read_coverage.py

@@ -12,7 +12,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import tempfile
+
 import coverage
+import xmltodict
 
 
 class DevNull(object):
@@ -26,10 +29,30 @@ class ReadCoverage(object):
     def __init__(self, coverage_file=None):
         self.cov = coverage.Coverage(data_file=coverage_file)
         self.cov.load()
-        self.cov_pct = self.cov.report(file=DevNull())
+        xmlfile = tempfile.NamedTemporaryFile(suffix='.xml')
+        self.cov_pct = self.cov.xml_report(outfile=xmlfile.name)
+        self.covdoc = xmltodict.parse(xmlfile.read())
+        xmlfile.close()
 
     def get_data(self):
         return self.cov.get_data()
 
     def get_coverage_rate(self):
         return self.cov_pct / 100
+
+    def get_rates_by_files(self):
+        rates = []
+        for p in self.covdoc['coverage']['packages']['package']:
+            try:
+                # FIXME(masayukig): Try to access the first element. This is
+                # ugly..
+                p['classes']['class'][0]
+                for c in p['classes']['class']:
+                    rates.append({'filename': c['@filename'],
+                                  'line-rate': c['@line-rate']})
+            except KeyError:
+                # NOTE(masayukig): This has only one class
+                c = p['classes']['class']
+                rates.append({'filename': c['@filename'],
+                              'line-rate': c['@line-rate']})
+        return rates

coverage2sql/shell.py

@@ -62,9 +62,11 @@ def parse_args(argv, default_config_files=None):
          default_config_files=default_config_files)
 
 
-def process_results(project_name=".", coverage_rate=0.0):
+def process_results(project_name=".", coverage_rate=0.0, rates=[]):
     session = api.get_session()
-    api.create_coverage(project_name, coverage_rate, test_type=CONF.test_type)
+    cov = api.create_coverage(project_name, coverage_rate, rates,
+                              test_type=CONF.test_type, session=session)
+    api.add_file_rates(cov.id, rates, session)
     session.close()
 
 
@@ -76,9 +78,10 @@ def main():
     if CONF.coverage_file:
         cov = coverage.ReadCoverage(CONF.coverage_file)
         coverage_rate = cov.get_coverage_rate()
+        rates = cov.get_rates_by_files()
     else:
         raise NotImplementedError()
-    process_results(project_name, coverage_rate)
+    process_results(project_name, coverage_rate, rates)
 
 
 if __name__ == "__main__":

coverage2sql/tests/db/test_api.py

@@ -65,3 +65,14 @@ class TestDatabaseAPI(base.TestCase):
         self.assertTrue(covs is not None)
         self.assertEqual(len(covs), 1)
         self.assertEqual(covs[0].project_name, 'foo1_project')
+
+    def test_add_file_rates(self):
+        rates = []
+        rates.append({'filename': 'foo/bar0', 'line-rate': '0'})
+        rates.append({'filename': 'foo/bar1', 'line-rate': '1'})
+        rates.append({'filename': 'foo/bar2', 'line-rate': '0.92'})
+        files = api.add_file_rates(1, rates)
+        self.assertEqual(3, len(files))
+        for r, f in zip(rates, files):
+            self.assertEqual(r['filename'], f.filename)
+            self.assertEqual(r['line-rate'], f.line_rate)

coverage2sql/tests/test_shell.py

@@ -44,6 +44,10 @@ class TestMain(base.TestCase):
             'get_coverage_rate')
         fake_read_coverage.get_coverage_rate.return_value = (
             fake_get_coverage_rate)
+        fake_read_coverage.get_rates_by_files = mock.MagicMock(
+            'get_rates_by_files')
+        fake_read_coverage.get_rates_by_files.return_value = (
+            {'filename': 'foo/bar.py', 'line-rate': '0.99'})
         read_coverage_mock.return_value = fake_read_coverage
         shell.main()
         read_coverage_mock.assert_called_with(mock.ANY)

releasenotes/notes/add-files-table-e60ac2e98b5f543b.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Add 'Files' table to store individual coverage data by file.

releasenotes/source/index.rst

@@ -1,14 +1,13 @@
 Welcome to Coverage2sql Release Notes documentation!
-===================================================
+====================================================
 
 Contents
 ========
 
-.. toctree::
-   :maxdepth: 2
-
-   unreleased
+ .. toctree::
+    :maxdepth: 1
 
+    unreleased
 
 Indices and tables
 ==================

requirements.txt

@@ -6,3 +6,5 @@ pbr>=1.6
 SQLAlchemy>=0.8.2
 alembic>=0.4.1
 oslo.config>=1.4.0.0a3
+coverage>=4.3
+xmltodict>=0.10.1  # MIT

test-requirements.txt

@@ -4,7 +4,6 @@
 
 hacking<0.11,>=0.10.0
 
-coverage>=3.6
 docutils>=0.11  # OSI-Approved Open Source, Public Domain
 fixtures>=0.3.14
 python-subunit>=0.0.18