Move tests under stackalytics package

Change-Id: I8475c31bb6120e8b07381217e232c861a8407053

stackalytics/tests/api/test_api.py

@@ -0,0 +1,179 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+import itertools
+import json
+import uuid
+
+import mock
+import six
+import testtools
+
+from stackalytics.dashboard import web
+from stackalytics.processor import runtime_storage
+
+
+class TestAPI(testtools.TestCase):
+
+    def setUp(self):
+        super(TestAPI, self).setUp()
+        self.app = web.app.test_client()
+
+
+@contextlib.contextmanager
+def make_runtime_storage(data, *generators):
+    _add_generated_records(data, *generators)
+
+    runtime_storage_inst = TestStorage(data)
+    setattr(web.app, 'stackalytics_vault', None)
+
+    with mock.patch('stackalytics.processor.runtime_storage.'
+                    'get_runtime_storage') as get_runtime_storage_mock:
+        get_runtime_storage_mock.return_value = runtime_storage_inst
+        try:
+            yield runtime_storage_inst
+        finally:
+            pass
+
+
+def make_records(**kwargs):
+    GENERATORS = {
+        'commit': _generate_commits,
+        'mark': _generate_marks,
+        'review': _generate_review,
+    }
+
+    def generate_records():
+        for record_type in kwargs.get('record_type', []):
+            if record_type in GENERATORS.keys():
+                for values in algebraic_product(**kwargs):
+                    record = next(GENERATORS[record_type]())
+                    record.update(values)
+                    yield record
+
+    return generate_records
+
+
+def make_module(module_name):
+    return {'id': module_name,
+            'module_group_name': module_name,
+            'modules': [module_name],
+            'tag': 'module'}
+
+
+class TestStorage(runtime_storage.RuntimeStorage):
+
+    def __init__(self, data):
+        super(TestStorage, self).__init__('test://')
+        self.data = data
+
+    def get_update(self, pid):
+        for record in self.get_all_records():
+            yield record
+
+    def get_by_key(self, key):
+        return self.data.get(key)
+
+    def set_by_key(self, key, value):
+        super(TestStorage, self).set_by_key(key, value)
+
+    def get_all_records(self):
+        for n in range(self.get_by_key('record:count') or 0):
+            record = self.get_by_key('record:%s' % n)
+            if record:
+                yield record
+
+
+def _generate_commits():
+        commit = {
+            'commit_id': str(uuid.uuid4()),
+            'lines_added': 9, 'module': 'nova', 'record_type': 'commit',
+            'message': 'Closes bug 1212953\n\nChange-Id: '
+                       'I33f0f37b6460dc494abf2520dc109c9893ace9e6\n',
+            'subject': 'Fixed affiliation of Edgar and Sumit', 'loc': 10,
+            'user_id': 'john_doe',
+            'primary_key': str(uuid.uuid4()),
+            'author_email': 'john_doe@ibm.com', 'company_name': 'IBM',
+            'lines_deleted': 1, 'week': 2275,
+            'blueprint_id': None, 'bug_id': u'1212953',
+            'files_changed': 1, 'author_name': u'John Doe',
+            'date': 1376737923, 'launchpad_id': u'john_doe',
+            'branches': set([u'master']),
+            'change_id': u'I33f0f37b6460dc494abf2520dc109c9893ace9e6',
+            'release': u'icehouse'
+        }
+        yield commit
+
+
+def _generate_marks():
+        mark = {
+            'launchpad_id': 'john_doe', 'week': 2294, 'user_id': 'john_doe',
+            'description': 'Approved', 'author_name': 'John Doe',
+            'author_email': 'john_doe@gmail.com',
+            'primary_key': str(uuid.uuid4()) + 'Workflow',
+            'module': 'glance', 'patch': 2, 'record_type': 'mark',
+            'company_name': '*independent', 'branch': 'master',
+            'date': 1387860458, 'record_id': 37184, 'release': 'icehouse',
+            'value': 1, 'type': 'Workflow',
+            'review_id': str(uuid.uuid4())}
+        yield mark
+
+
+def _generate_review():
+    yield {
+        'status': 'NEW', 'review_number': 6, 'number': '60721',
+        'module': 'glance', 'topic': 'bug/1258999', 'record_type': 'review',
+        'value': -2, 'open': True,
+        'id': str(uuid.uuid4()),
+        'subject': 'Adding missing copy_from policy from policy.json',
+        'user_id': 'john_doe',
+        'primary_key': 'Ibc0d1fa7626629c28c514514a985a6b89db2ac69',
+        'author_email': 'john_doe@gmail.com', 'company_name': '*independent',
+        'branch': 'master',
+        'launchpad_id': 'john_doe', 'lastUpdated': 1387865203,
+        'author_name': 'John Doe', 'date': 1386547707,
+        'url': 'https://review.openstack.org/60721',
+        'sortKey': '0029f92e0000ed31', 'project': 'openstack/glance',
+        'week': 2292, 'release': 'icehouse', 'updated_on': 1387865147
+    }
+
+
+def _add_generated_records(data, *generators):
+    count = 0
+    for gen in generators:
+        for record in gen():
+            record['record_id'] = count
+            data['record:%s' % count] = record
+            count += 1
+    data['record:count'] = count
+
+
+def algebraic_product(**kwargs):
+    position_to_key = {}
+    values = []
+    for key, value in six.iteritems(kwargs):
+        position_to_key[len(values)] = key
+        values.append(value)
+
+    for chain in itertools.product(*values):
+        result = {}
+        for position, key in six.iteritems(position_to_key):
+            result[key] = chain[position]
+        yield result
+
+
+def load_json(api_response):
+    return json.loads(api_response.data.decode('utf8'))

stackalytics/tests/api/test_companies.py

@@ -0,0 +1,111 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from stackalytics.tests.api import test_api
+
+
+class TestAPICompanies(test_api.TestAPI):
+
+    def test_get_companies(self):
+        with test_api.make_runtime_storage(
+                {
+                    'repos': [
+                        {'module': 'nova', 'project_type': 'openstack',
+                         'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/nova.git'},
+                        {'module': 'glance', 'project_type': 'openstack',
+                         'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/glance.git'}
+                    ],
+                    'project_types': [
+                        {'id': 'openstack', 'title': 'OpenStack',
+                         'modules': ['nova', 'glance']}],
+                    'releases': [{'release_name': 'prehistory',
+                                  'end_date': 1234567890},
+                                 {'release_name': 'icehouse',
+                                  'end_date': 1234567890}],
+                    'module_groups': {
+                        'openstack': {'module_group_name': 'openstack',
+                                      'modules': ['nova', 'glance']},
+                        'nova': {'module_group_name': 'nova',
+                                 'modules': ['nova']},
+                        'glance': {'module_group_name': 'glance',
+                                   'modules': ['glance']},
+                    }},
+                test_api.make_records(record_type=['commit'],
+                                      loc=[10, 20, 30],
+                                      module=['glance'],
+                                      company_name=['NEC', 'IBM', 'NTT']),
+                test_api.make_records(record_type=['review'],
+                                      primary_key=['0123456789', '9876543210'],
+                                      module=['glance'],
+                                      company_name=['IBM']),
+                test_api.make_records(record_type=['mark'],
+                                      review_id=['0123456789', '9876543210'],
+                                      module=['glance'],
+                                      company_name=['IBM']),
+                test_api.make_records(record_type=['mark'],
+                                      review_id=['0123456789'],
+                                      module=['glance'],
+                                      company_name=['NEC'])):
+
+            response = self.app.get('/api/1.0/companies?metric=commits&'
+                                    'module=glance')
+            companies = test_api.load_json(response)['data']
+            self.assertEqual([{'id': 'ibm', 'text': 'IBM'},
+                              {'id': 'nec', 'text': 'NEC'},
+                              {'id': 'ntt', 'text': 'NTT'}], companies)
+
+            response = self.app.get('/api/1.0/companies?metric=marks&'
+                                    'module=glance')
+            companies = test_api.load_json(response)['data']
+            self.assertEqual([{'id': 'ibm', 'text': 'IBM'},
+                              {'id': 'nec', 'text': 'NEC'}], companies)
+
+    def test_get_company(self):
+        with test_api.make_runtime_storage(
+                {
+                    'repos': [
+                        {'module': 'nova', 'project_type': 'openstack',
+                         'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/nova.git'},
+                        {'module': 'glance', 'project_type': 'openstack',
+                         'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/glance.git'}
+                    ],
+                    'module_groups': {
+                        'nova': test_api.make_module('nova'),
+                        'glance': test_api.make_module('glance'),
+                    },
+                    'releases': [{'release_name': 'prehistory',
+                                  'end_date': 1234567890},
+                                 {'release_name': 'icehouse',
+                                  'end_date': 1234567890}],
+                    'project_types': [
+                        {'id': 'all', 'title': 'All',
+                         'modules': ['nova', 'glance', 'nova-cli']},
+                        {'id': 'openstack', 'title': 'OpenStack',
+                         'modules': ['nova', 'glance']}]},
+                test_api.make_records(record_type=['commit'],
+                                      loc=[10, 20, 30],
+                                      module=['glance'],
+                                      company_name=['NEC', 'IBM', 'NTT'])):
+
+            response = self.app.get('/api/1.0/companies/nec?module=glance')
+            company = test_api.load_json(response)['company']
+            self.assertEqual({'id': 'nec', 'text': 'NEC'}, company)
+
+            response = self.app.get('/api/1.0/companies/google?module=glance')
+            self.assertEqual(404, response.status_code)

stackalytics/tests/api/test_modules.py

@@ -0,0 +1,120 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from stackalytics.tests.api import test_api
+
+
+class TestAPIModules(test_api.TestAPI):
+
+    def test_get_modules(self):
+        with test_api.make_runtime_storage(
+                {
+                    'repos': [
+                        {'module': 'nova', 'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/nova.git'},
+                        {'module': 'glance', 'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/glance.git'}
+                    ],
+                    'module_groups': {
+                        'nova-group': {'id': 'nova-group',
+                                       'module_group_name': 'nova-group',
+                                       'modules': ['nova', 'nova-cli'],
+                                       'tag': 'group'},
+                        'nova': test_api.make_module('nova'),
+                        'nova-cli': test_api.make_module('nova-cli'),
+                        'glance': test_api.make_module('glance'),
+                    },
+                    'releases': [
+                        {'release_name': 'prehistory', 'end_date': 1234567890},
+                        {'release_name': 'icehouse', 'end_date': 1234567890}],
+                    'project_types': [{'id': 'all', 'title': 'All',
+                                       'modules': ['nova', 'glance',
+                                                   'nova-cli']},
+                                      {'id': 'integrated',
+                                       'title': 'Integrated',
+                                       'modules': ['nova', 'glance']}]},
+                test_api.make_records(record_type=['commit'],
+                                      module=['glance', 'nova', 'nova-cli'])):
+
+            response = self.app.get('/api/1.0/modules?'
+                                    'project_type=all&metric=commits')
+            modules = test_api.load_json(response)['data']
+            self.assertEqual(
+                [{'id': 'glance', 'text': 'glance', 'tag': 'module'},
+                 {'id': 'nova', 'text': 'nova', 'tag': 'module'},
+                 {'id': 'nova-cli', 'text': 'nova-cli', 'tag': 'module'},
+                 {'id': 'nova-group', 'text': 'nova-group', 'tag': 'group'}],
+                modules,
+                message='Expected modules belonging to project type plus '
+                        'module groups that are completely within '
+                        'project type')
+
+            response = self.app.get('/api/1.0/modules?module=nova-group&'
+                                    'project_type=integrated&metric=commits')
+            modules = test_api.load_json(response)['data']
+            self.assertEqual(
+                [{'id': 'glance', 'text': 'glance', 'tag': 'module'},
+                 {'id': 'nova', 'text': 'nova', 'tag': 'module'},
+                 {'id': 'nova-group', 'text': 'nova-group', 'tag': 'group'}],
+                modules,
+                message='Expected modules belonging to project type plus '
+                        'module groups that are completely within '
+                        'project type')
+
+    def test_get_module(self):
+        with test_api.make_runtime_storage(
+                {
+                    'repos': [
+                        {'module': 'nova', 'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/nova.git'}],
+                    'module_groups': {
+                        'nova-group': {'id': 'nova-group',
+                                       'module_group_name': 'nova-group',
+                                       'modules': ['nova-cli', 'nova'],
+                                       'tag': 'group'},
+                        'nova': test_api.make_module('nova'),
+                        'nova-cli': test_api.make_module('nova-cli'),
+                    },
+                    'releases': [{'release_name': 'prehistory',
+                                  'end_date': 1234567890},
+                                 {'release_name': 'icehouse',
+                                  'end_date': 1234567890}],
+                    'project_types': [
+                        {'id': 'all', 'title': 'All',
+                         'modules': ['nova', 'glance', 'nova-cli']},
+                        {'id': 'openstack', 'title': 'OpenStack',
+                         'modules': ['nova', 'glance']}]},
+                test_api.make_records(record_type=['commit'])):
+
+            response = self.app.get('/api/1.0/modules/nova')
+            module = test_api.load_json(response)['module']
+            self.assertEqual(
+                {'id': 'nova',
+                 'modules': [
+                     {'module_name': 'nova',
+                      'repo_uri': 'git://git.openstack.org/openstack/nova.git'}
+                 ],
+                 'name': 'Nova', 'tag': 'module'}, module)
+
+            response = self.app.get('/api/1.0/modules/nova-group')
+            module = test_api.load_json(response)['module']
+            self.assertEqual(
+                {'id': 'nova-group',
+                 'modules': [
+                     {'module_name': 'nova-cli'},
+                     {'module_name': 'nova',
+                      'repo_uri': 'git://git.openstack.org/openstack/nova.git'}
+                 ],
+                 'name': 'Nova-group', 'tag': 'group'}, module)

stackalytics/tests/api/test_releases.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from stackalytics.tests.api import test_api
+
+
+class TestAPIReleases(test_api.TestAPI):
+
+    def test_releases(self):
+        with test_api.make_runtime_storage(
+                {'releases': [
+                    {'release_name': 'prehistory', 'end_date': 1365033600},
+                    {'release_name': 'havana', 'end_date': 1381968000},
+                    {'release_name': 'icehouse', 'end_date': 1397692800}],
+                 'project_types': [
+                     {'id': 'all', 'title': 'All',
+                      'modules': ['nova', 'glance', 'nova-cli']},
+                     {'id': 'openstack', 'title': 'OpenStack',
+                      'modules': ['nova', 'glance']}]},
+                test_api.make_records(record_type=['commit'])):
+            response = self.app.get('/api/1.0/releases')
+            releases = test_api.load_json(response)['data']
+            self.assertEqual(3, len(releases))
+            self.assertIn({'id': 'all', 'text': 'All'}, releases)
+            self.assertIn({'id': 'icehouse', 'text': 'Icehouse'}, releases)

stackalytics/tests/api/test_stats.py

@@ -0,0 +1,178 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from stackalytics.tests.api import test_api
+
+
+class TestAPIStats(test_api.TestAPI):
+
+    def test_get_modules(self):
+        with test_api.make_runtime_storage(
+                {
+                    'repos': [
+                        {'module': 'nova', 'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/nova.git'},
+                        {'module': 'glance', 'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/glance.git'}
+                    ],
+                    'releases': [{'release_name': 'prehistory',
+                                  'end_date': 1234567890},
+                                 {'release_name': 'icehouse',
+                                  'end_date': 1234567890}],
+                    'module_groups': {
+                        'openstack': {'id': 'openstack',
+                                      'module_group_name': 'openstack',
+                                      'modules': ['nova', 'glance'],
+                                      'tag': 'group'},
+                        'nova': test_api.make_module('nova'),
+                        'glance': test_api.make_module('glance'),
+                    },
+                    'project_types': [
+                        {'id': 'all', 'title': 'All',
+                         'modules': ['nova', 'glance']}]},
+                test_api.make_records(record_type=['commit'],
+                                      loc=[10, 20, 30],
+                                      module=['nova']),
+                test_api.make_records(record_type=['commit'],
+                                      loc=[100, 200, 300],
+                                      module=['glance'])):
+            response = self.app.get('/api/1.0/stats/modules?metric=loc&'
+                                    'project_type=all')
+            stats = test_api.load_json(response)['stats']
+            self.assertEqual(2, len(stats))
+            self.assertEqual(600, stats[0]['metric'])
+            self.assertEqual('glance', stats[0]['id'])
+            self.assertEqual(60, stats[1]['metric'])
+            self.assertEqual('nova', stats[1]['id'])
+
+    def test_get_engineers(self):
+        with test_api.make_runtime_storage(
+                {
+                    'repos': [
+                        {'module': 'nova', 'project_type': 'openstack',
+                         'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/nova.git'},
+                        {'module': 'glance', 'project_type': 'openstack',
+                         'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/glance.git'}
+                    ],
+                    'releases': [{'release_name': 'prehistory',
+                                  'end_date': 1234567890},
+                                 {'release_name': 'icehouse',
+                                  'end_date': 1234567890}],
+                    'module_groups': {
+                        'openstack': {'id': 'openstack',
+                                      'module_group_name': 'openstack',
+                                      'modules': ['nova', 'glance'],
+                                      'tag': 'group'},
+                        'nova': test_api.make_module('nova'),
+                        'glance': test_api.make_module('glance'),
+                    },
+                    'project_types': [
+                        {'id': 'all', 'title': 'All',
+                         'modules': ['nova', 'glance']}],
+                    'user:john_doe': {
+                        'seq': 1, 'user_id': 'john_doe',
+                        'user_name': 'John Doe',
+                        'companies': [{'company_name': 'NEC', 'end_date': 0}],
+                        'emails': ['john_doe@gmail.com'], 'core': []},
+                    'user:bill': {
+                        'seq': 1, 'user_id': 'bill', 'user_name': 'Bill Smith',
+                        'companies': [{'company_name': 'IBM', 'end_date': 0}],
+                        'emails': ['bill_smith@gmail.com'], 'core': []}},
+                test_api.make_records(record_type=['commit'],
+                                      loc=[10, 20, 30],
+                                      module=['nova'],
+                                      user_id=['john_doe']),
+                test_api.make_records(record_type=['commit'],
+                                      loc=[100, 200, 300],
+                                      module=['glance'],
+                                      user_id=['john_doe']),
+                test_api.make_records(record_type=['review'],
+                                      primary_key=['0123456789'],
+                                      module=['glance']),
+                test_api.make_records(record_type=['mark'],
+                                      review_id=['0123456789'],
+                                      module=['glance'],
+                                      user_id=['john_doe', 'bill'])):
+            response = self.app.get('/api/1.0/stats/engineers?metric=loc&'
+                                    'project_type=all')
+            stats = test_api.load_json(response)['stats']
+            self.assertEqual(1, len(stats))
+            self.assertEqual(660, stats[0]['metric'])
+
+    def test_get_engineers_extended(self):
+        with test_api.make_runtime_storage(
+                {
+                    'repos': [
+                        {'module': 'nova', 'project_type': 'openstack',
+                         'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/nova.git'},
+                        {'module': 'glance', 'project_type': 'openstack',
+                         'organization': 'openstack',
+                         'uri': 'git://git.openstack.org/openstack/glance.git'}
+                    ],
+                    'releases': [{'release_name': 'prehistory',
+                                  'end_date': 1234567890},
+                                 {'release_name': 'icehouse',
+                                  'end_date': 1234567890}],
+                    'module_groups': {
+                        'openstack': {'id': 'openstack',
+                                      'module_group_name': 'openstack',
+                                      'modules': ['nova', 'glance'],
+                                      'tag': 'group'},
+                        'nova': test_api.make_module('nova'),
+                        'glance': test_api.make_module('glance'),
+                    },
+                    'project_types': [
+                        {'id': 'all', 'title': 'All',
+                         'modules': ['nova', 'glance']}],
+                    'user:john_doe': {
+                        'seq': 1, 'user_id': 'john_doe',
+                        'user_name': 'John Doe',
+                        'companies': [{'company_name': 'NEC', 'end_date': 0}],
+                        'emails': ['john_doe@gmail.com'], 'core': []},
+                    'user:smith': {
+                        'seq': 1, 'user_id': 'smith',
+                        'user_name': 'Bill Smith',
+                        'companies': [{'company_name': 'IBM', 'end_date': 0}],
+                        'emails': ['bill_smith@gmail.com'], 'core': []}},
+                test_api.make_records(record_type=['commit'],
+                                      loc=[10, 20, 30],
+                                      module=['nova'],
+                                      user_id=['john_doe']),
+                test_api.make_records(record_type=['review'],
+                                      primary_key=['0123456789', '9876543210'],
+                                      module=['glance']),
+                test_api.make_records(record_type=['mark'],
+                                      review_id=['0123456789', '9876543210'],
+                                      module=['glance'],
+                                      value=[1],
+                                      type=['Code-Review'],
+                                      author_name=['John Doe'],
+                                      user_id=['john_doe']),
+                test_api.make_records(record_type=['mark'],
+                                      review_id=['0123456789'],
+                                      module=['glance'],
+                                      author_name=['Bill Smith'],
+                                      user_id=['smith'])):
+            response = self.app.get('/api/1.0/stats/engineers_extended?'
+                                    'project_type=all')
+            stats = test_api.load_json(response)['stats']
+            self.assertEqual(2, len(stats))
+            self.assertEqual(2, stats[0]['mark'])
+            self.assertEqual('john_doe', stats[0]['id'])
+            self.assertEqual(3, stats[0]['commit'])
+            self.assertEqual(2, stats[0]['1'])

stackalytics/tests/api/test_users.py

@@ -0,0 +1,74 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from stackalytics.tests.api import test_api
+
+
+class TestAPIUsers(test_api.TestAPI):
+
+    def test_users(self):
+        with test_api.make_runtime_storage(
+                {'repos': [
+                    {'module': 'nova', 'organization': 'openstack',
+                     'uri': 'git://git.openstack.org/openstack/nova.git'}],
+                 'project_types': [
+                     {'id': 'openstack', 'title': 'openstack',
+                      'modules': ['nova', 'glance']}],
+                 'releases': [{'release_name': 'prehistory',
+                               'end_date': 1234567890},
+                              {'release_name': 'icehouse',
+                               'end_date': 1234567890}],
+                 'module_groups': {
+                     'nova': test_api.make_module('nova'),
+                     'glance': test_api.make_module('glance')},
+                 'user:john_doe': {'user_name': 'John Doe'},
+                 'user:bill_smith': {'user_name': 'Bill Smith'}},
+                test_api.make_records(record_type=['commit'], module=['nova'],
+                                      user_id=['john_doe', 'bill_smith'])):
+            response = self.app.get('/api/1.0/users?'
+                                    'module=nova&metric=commits')
+            users = test_api.load_json(response)['data']
+            self.assertEqual(2, len(users))
+            self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users)
+            self.assertIn({'id': 'bill_smith', 'text': 'Bill Smith'}, users)
+
+    def test_user_details(self):
+        with test_api.make_runtime_storage(
+                {'user:john_doe': {
+                    'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe',
+                    'companies': [{'company_name': 'NEC', 'end_date': 0}],
+                    'emails': 'john_doe@gmail.com'}},
+                test_api.make_records(record_type=['commit'], module=['nova'],
+                                      user_name=['John Doe', 'Bill Smith'])):
+            response = self.app.get('/api/1.0/users/john_doe')
+            user = test_api.load_json(response)['user']
+            self.assertEqual('john_doe', user['user_id'])
+
+    def test_user_not_found(self):
+        with test_api.make_runtime_storage(
+                {'user:john_doe': {
+                    'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe',
+                    'companies': [{'company_name': 'NEC', 'end_date': 0}],
+                    'emails': 'john_doe@gmail.com'},
+                 'repos': [
+                     {'module': 'nova', 'organization': 'openstack',
+                      'uri': 'git://git.openstack.org/openstack/nova.git'}],
+                 'module_groups': {'openstack': {
+                     'module_group_name': 'openstack',
+                     'modules': ['nova']}}},
+                test_api.make_records(record_type=['commit'], module=['nova'],
+                                      user_name=['John Doe', 'Bill Smith'])):
+            response = self.app.get('/api/1.0/users/nonexistent')
+            self.assertEqual(404, response.status_code)

stackalytics/tests/unit/__init__.py

@@ -0,0 +1 @@
+__author__ = 'ishakhat'

stackalytics/tests/unit/test_bps.py

@@ -0,0 +1,221 @@
+# Copyright (c) 2015 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import mock
+import testtools
+
+from stackalytics.processor import bps
+
+
+BUG = json.loads("""
+{
+    "date_closed": "2015-06-02T17:31:05.820479+00:00",
+    "date_assigned": "2015-06-02T17:31:44.957976+00:00",
+    "title": "Bug #1458945 in Sahara: \\\"Use graduated oslo.policy\\\"",
+    "bug_link": "https://api.launchpad.net/devel/bugs/1458945",
+    "bug_watch_link": null,
+    "milestone_link": null,
+    "date_left_closed": null,
+    "date_fix_committed": "2015-06-02T17:31:05.820479+00:00",
+    "date_fix_released": "2015-06-02T17:31:05.820479+00:00",
+    "date_in_progress": "2015-06-02T17:31:05.820479+00:00",
+    "resource_type_link": "https://api.launchpad.net/devel/#bug_task",
+    "status": "Fix Released",
+    "bug_target_name": "sahara",
+    "importance": "Medium",
+    "assignee_link": "https://api.launchpad.net/devel/~slukjanov",
+    "date_triaged": "2015-06-02T17:31:05.820479+00:00",
+    "self_link": "https://api.launchpad.net/devel/sahara/+bug/1458945",
+    "target_link": "https://api.launchpad.net/devel/sahara",
+    "bug_target_display_name": "Sahara",
+    "related_tasks_collection_link":
+      "https://api.launchpad.net/devel/sahara/+bug/1458945/related_tasks",
+    "date_confirmed": "2015-06-02T17:31:05.820479+00:00",
+    "date_left_new": "2015-06-02T17:31:05.820479+00:00",
+    "web_link": "https://bugs.launchpad.net/sahara/+bug/1458945",
+    "owner_link": "https://api.launchpad.net/devel/~samueldmq",
+    "date_created": "2015-06-02T13:35:54.101235+00:00",
+    "date_incomplete": null,
+    "is_complete": true
+}
+""")
+
+ANOTHER_MILESTONE_BUG = json.loads("""
+{
+    "date_closed": "2015-06-02T17:31:05.820479+00:00",
+    "date_assigned": "2015-06-02T17:31:44.957976+00:00",
+    "title": "Bug #1458945 in Sahara Kilo: \\\"Use graduated oslo.policy\\\"",
+    "bug_link": "https://api.launchpad.net/devel/bugs/1458945",
+    "bug_watch_link": null,
+    "milestone_link": null,
+    "date_left_closed": null,
+    "date_fix_committed": "2015-06-02T17:31:05.820479+00:00",
+    "date_fix_released": "2015-06-02T17:31:05.820479+00:00",
+    "date_in_progress": "2015-06-02T17:31:05.820479+00:00",
+    "resource_type_link": "https://api.launchpad.net/devel/#bug_task",
+    "status": "Fix Released",
+    "bug_target_name": "sahara/kilo",
+    "importance": "Medium",
+    "assignee_link": "https://api.launchpad.net/devel/~slukjanov",
+    "date_triaged": "2015-06-02T17:31:05.820479+00:00",
+    "self_link": "https://api.launchpad.net/devel/sahara/kilo/+bug/1458945",
+    "target_link": "https://api.launchpad.net/devel/sahara/kilo",
+    "bug_target_display_name": "Sahara Kilo",
+    "related_tasks_collection_link":
+      "https://api.launchpad.net/devel/sahara/kilo/+bug/1458945/related_tasks",
+    "date_confirmed": "2015-06-02T17:31:05.820479+00:00",
+    "date_left_new": "2015-06-02T17:31:05.820479+00:00",
+    "web_link": "https://bugs.launchpad.net/sahara/kilo/+bug/1458945",
+    "owner_link": "https://api.launchpad.net/devel/~samueldmq",
+    "date_created": "2015-06-02T13:35:54.101235+00:00",
+    "date_incomplete": null,
+    "is_complete": true
+}
+""")
+
+LINKED_BUG = json.loads("""
+{
+    "date_closed": "2015-06-24T20:59:57.982386+00:00",
+    "date_assigned": "2015-06-18T06:46:03.741208+00:00",
+    "title": "Bug #1458945 in Barbican: \\\"Use graduated oslo.policy\\\"",
+    "bug_link": "https://api.launchpad.net/devel/bugs/1458945",
+    "bug_watch_link": null,
+    "milestone_link":
+      "https://api.launchpad.net/devel/barbican/+milestone/liberty-1",
+    "date_left_closed": null,
+    "date_fix_committed": "2015-06-18T06:45:39.997949+00:00",
+    "date_fix_released": "2015-06-24T20:59:57.982386+00:00",
+    "date_in_progress": "2015-06-18T06:45:39.997949+00:00",
+    "resource_type_link": "https://api.launchpad.net/devel/#bug_task",
+    "status": "Fix Released",
+    "bug_target_name": "barbican",
+    "importance": "Medium",
+    "assignee_link": "https://api.launchpad.net/devel/~juan-osorio-robles",
+    "date_triaged": "2015-06-18T06:45:39.997949+00:00",
+    "self_link": "https://api.launchpad.net/devel/barbican/+bug/1458945",
+    "target_link": "https://api.launchpad.net/devel/barbican",
+    "bug_target_display_name": "Barbican",
+    "related_tasks_collection_link":
+      "https://api.launchpad.net/devel/barbican/+bug/1458945/related_tasks",
+    "date_confirmed": "2015-06-18T06:45:39.997949+00:00",
+    "date_left_new": "2015-06-18T06:45:39.997949+00:00",
+    "web_link": "https://bugs.launchpad.net/barbican/+bug/1458945",
+    "owner_link": "https://api.launchpad.net/devel/~samueldmq",
+    "date_created": "2015-05-26T17:47:32.438795+00:00",
+    "date_incomplete": null,
+    "is_complete": true
+}
+""")
+
+
+class TestBps(testtools.TestCase):
+    def setUp(self):
+        super(TestBps, self).setUp()
+        p_module_exists = mock.patch(
+            'stackalytics.processor.launchpad_utils.lp_module_exists')
+        m_module_exists = p_module_exists.start()
+        m_module_exists.return_value = True
+
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
+    def test_log(self, lp_bug_generator):
+        repo = {
+            'module': 'sahara'
+        }
+        modified_since = 1234567890
+        lp_bug_generator.return_value = iter([BUG])
+
+        expected = [{
+            'assignee': 'slukjanov',
+            'date_created': 1433252154,
+            'date_fix_committed': 1433266265,
+            'id': 'sahara/1458945',
+            'importance': 'Medium',
+            'module': 'sahara',
+            'owner': 'samueldmq',
+            'status': 'Fix Released',
+            'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"',
+            'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945'
+        }]
+
+        actual = list(bps.log(repo, modified_since))
+
+        self.assertEqual(expected, actual)
+
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
+    def test_log_additional_module(self, lp_bug_generator):
+        # bug linked to another project should not appear
+        repo = {
+            'module': 'sahara'
+        }
+        modified_since = 1234567890
+        lp_bug_generator.return_value = iter([BUG, LINKED_BUG])
+
+        expected = [{
+            'assignee': 'slukjanov',
+            'date_created': 1433252154,
+            'date_fix_committed': 1433266265,
+            'id': 'sahara/1458945',
+            'importance': 'Medium',
+            'module': 'sahara',
+            'owner': 'samueldmq',
+            'status': 'Fix Released',
+            'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"',
+            'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945'
+        }]
+
+        actual = list(bps.log(repo, modified_since))
+
+        self.assertEqual(expected, actual)
+
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
+    def test_log_additional_milestone(self, lp_bug_generator):
+        # bug linked to different milestone should be mapped to the release
+        repo = {
+            'module': 'sahara'
+        }
+        modified_since = 1234567890
+        lp_bug_generator.return_value = iter([BUG, ANOTHER_MILESTONE_BUG])
+
+        expected = [{
+            'assignee': 'slukjanov',
+            'date_created': 1433252154,
+            'date_fix_committed': 1433266265,
+            'id': 'sahara/1458945',
+            'importance': 'Medium',
+            'module': 'sahara',
+            'owner': 'samueldmq',
+            'status': 'Fix Released',
+            'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"',
+            'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945'
+        }, {
+            'assignee': 'slukjanov',
+            'date_created': 1433252154,
+            'date_fix_committed': 1433266265,
+            'id': 'sahara/kilo/1458945',
+            'importance': 'Medium',
+            'module': 'sahara',
+            'release': 'kilo',
+            'owner': 'samueldmq',
+            'status': 'Fix Released',
+            'title': 'Bug #1458945 in Sahara Kilo: '
+                     '"Use graduated oslo.policy"',
+            'web_link': 'https://bugs.launchpad.net/sahara/kilo/+bug/1458945'
+
+        }]
+
+        actual = list(bps.log(repo, modified_since))
+
+        self.assertEqual(expected, actual)

stackalytics/tests/unit/test_config_files.py

@@ -0,0 +1,178 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import json
+
+import jsonschema
+import six
+import testtools
+
+
+def dict_raise_on_duplicates(ordered_pairs):
+    """Reject duplicate keys."""
+    d = {}
+    for k, v in ordered_pairs:
+        if k in d:
+            raise ValueError("duplicate key: %s (value: %s)" % (k, v))
+        else:
+            d[k] = v
+    return d
+
+
+class TestConfigFiles(testtools.TestCase):
+    def setUp(self):
+        super(TestConfigFiles, self).setUp()
+
+    def _read_raw_file(self, file_name):
+        if six.PY3:
+            opener = functools.partial(open, encoding='utf8')
+        else:
+            opener = open
+        with opener(file_name, 'r') as content_file:
+            return content_file.read()
+
+    def _read_file(self, file_name):
+        return json.loads(self._read_raw_file(file_name))
+
+    def _verify_ordering(self, array, key, msg):
+        comparator = lambda x, y: (x > y) - (x < y)
+
+        diff_msg = ''
+        for i in range(len(array) - 1):
+            if comparator(key(array[i]), key(array[i + 1])) > 0:
+                diff_msg = ('Order fails at index %(index)s, '
+                            'elements:\n%(first)s:\n%(second)s' %
+                            {'index': i, 'first': array[i],
+                             'second': array[i + 1]})
+                break
+        if diff_msg:
+            self.fail(msg + '\n' + diff_msg)
+
+    def test_corrections(self):
+        corrections = self._read_file('etc/corrections.json')
+        schema = self._read_file('etc/corrections.schema.json')
+        jsonschema.validate(corrections, schema)
+
+    def _verify_default_data_duplicate_keys(self, file_name):
+        try:
+            json.loads(self._read_raw_file(file_name),
+                       object_pairs_hook=dict_raise_on_duplicates)
+        except ValueError as ve:
+            self.fail(ve)
+
+    def test_default_data_duplicate_keys(self):
+        self._verify_default_data_duplicate_keys('etc/default_data.json')
+
+    def test_test_default_data_duplicate_keys(self):
+        self._verify_default_data_duplicate_keys('etc/test_default_data.json')
+
+    def _verify_default_data_by_schema(self, file_name):
+        default_data = self._read_file(file_name)
+        schema = self._read_file('etc/default_data.schema.json')
+        try:
+            jsonschema.validate(default_data, schema)
+        except jsonschema.ValidationError as e:
+            self.fail(e)
+
+    def test_default_data_schema_conformance(self):
+        self._verify_default_data_by_schema('etc/default_data.json')
+
+    def test_test_default_data_schema_conformance(self):
+        self._verify_default_data_by_schema('etc/test_default_data.json')
+
+    def _verify_companies_in_alphabetical_order(self, file_name):
+        companies = self._read_file(file_name)['companies']
+        self._verify_ordering(
+            companies, key=lambda x: x['domains'][0],
+            msg='List of companies should be ordered by the first domain')
+
+    def test_companies_in_alphabetical_order(self):
+        self._verify_companies_in_alphabetical_order('etc/default_data.json')
+
+    def test_companies_in_alphabetical_order_in_test_file(self):
+        self._verify_companies_in_alphabetical_order(
+            'etc/test_default_data.json')
+
+    def _verify_users_in_alphabetical_order(self, file_name):
+        users = self._read_file(file_name)['users']
+        self._verify_ordering(
+            users, key=lambda x: (x.get('launchpad_id') or x.get('ldap_id') or
+                                  x.get('github_id')),
+            msg='List of users should be ordered by launchpad id or ldap id '
+                'or github id')
+
+    def test_users_in_alphabetical_order(self):
+        self._verify_users_in_alphabetical_order('etc/default_data.json')
+
+    def test_users_in_alphabetical_order_in_test_file(self):
+        self._verify_users_in_alphabetical_order('etc/test_default_data.json')
+
+    def _check_collision(self, storage, user, field, field_name):
+        self.assertFalse(
+            field in storage,
+            'Duplicate %s %s, collision between: %s and %s'
+            % (field_name, field, storage[field], user))
+        storage[field] = user
+
+    def _verify_users_unique(self, file_name):
+        users = self._read_file(file_name)['users']
+        storage = {}
+        for user in users:
+            if user.get('launchpad_id'):
+                field = user['launchpad_id']
+                self.assertFalse(
+                    field in storage,
+                    'Duplicate launchpad_id %s, collision between: %s and %s'
+                    % (field, storage.get(field), user))
+                storage[field] = user
+
+            if user.get('gerrit_id'):
+                field = user['gerrit_id']
+                self.assertFalse(
+                    ('gerrit:%s' % field) in storage,
+                    'Duplicate gerrit_id %s, collision between: %s and %s'
+                    % (field, storage.get(field), user))
+                storage['gerrit:%s' % field] = user
+
+            for email in user['emails']:
+                self.assertFalse(
+                    email in storage,
+                    'Duplicate email %s, collision between: %s and %s'
+                    % (email, storage.get(email), user))
+                storage[email] = user
+
+    def test_users_unique_profiles(self):
+        self._verify_users_unique('etc/default_data.json')
+
+    def test_users_unique_profiles_in_test_file(self):
+        self._verify_users_unique('etc/test_default_data.json')
+
+    def _verify_default_data_whitespace_issues(self, file_name):
+        data = self._read_raw_file(file_name)
+        line_n = 1
+        for line in data.split('\n'):
+            msg = 'Whitespace issue in "%s", line %s: ' % (line, line_n)
+            self.assertTrue(line.find('\t') == -1, msg=msg + 'tab character')
+            self.assertEqual(line.rstrip(), line,
+                             message=msg + 'trailing spaces')
+            line_n += 1
+
+    def test_default_data_whitespace_issues(self):
+        self._verify_default_data_whitespace_issues('etc/default_data.json')
+
+    def test_test_default_data_whitespace_issues(self):
+        self._verify_default_data_whitespace_issues(
+            'etc/test_default_data.json')

stackalytics/tests/unit/test_data.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+DEFAULT_DATA = {
+    'users': [
+        {
+            'launchpad_id': 'john_doe',
+            'user_name': 'John Doe',
+            'emails': ['johndoe@gmail.com', 'jdoe@nec.com'],
+            'companies': [
+                {'company_name': '*independent', 'end_date': '2013-May-01'},
+                {'company_name': 'NEC', 'end_date': None},
+            ]
+        },
+        {
+            'launchpad_id': 'smith',
+            'user_name': 'Smith',
+            'emails': ['smith@gmail.com', 'smith@nec.com'],
+            'companies': [
+                {'company_name': 'IBM', 'end_date': '2013-May-01'},
+                {'company_name': 'NEC', 'end_date': '2014-Jun-01'}
+            ]
+        },
+        {
+            'launchpad_id': 'ivan_ivanov',
+            'user_name': 'Ivan Ivanov',
+            'emails': ['ivanivan@yandex.ru', 'iivanov@mirantis.com'],
+            'companies': [
+                {'company_name': 'Mirantis', 'end_date': None},
+            ]
+        }
+    ],
+    'companies': [
+        {
+            'company_name': '*independent',
+            'domains': ['']
+        },
+        {
+            'company_name': 'NEC',
+            'domains': ['nec.com', 'nec.co.jp']
+        },
+        {
+            'company_name': 'Mirantis',
+            'domains': ['mirantis.com', 'mirantis.ru']
+        },
+    ],
+    'repos': [
+        {
+            'branches': ['master'],
+            'module': 'stackalytics',
+            'project_type': 'stackforge',
+            'uri': 'git://git.openstack.org/stackforge/stackalytics.git'
+        }
+    ],
+    'releases': [
+        {
+            'release_name': 'prehistory',
+            'end_date': '2011-Apr-21'
+        },
+        {
+            'release_name': 'Havana',
+            'end_date': '2013-Oct-17'
+        }
+    ]
+}
+
+USERS = DEFAULT_DATA['users']
+REPOS = DEFAULT_DATA['repos']
+COMPANIES = DEFAULT_DATA['companies']
+RELEASES = DEFAULT_DATA['releases']

stackalytics/tests/unit/test_default_data_processor.py

@@ -0,0 +1,99 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+
+import mock
+import testtools
+
+from stackalytics.processor import default_data_processor
+from stackalytics.processor import normalizer
+from stackalytics.tests.unit import test_data
+
+
+class TestDefaultDataProcessor(testtools.TestCase):
+    def setUp(self):
+        super(TestDefaultDataProcessor, self).setUp()
+
+        self.get_users = mock.Mock(return_value=[
+            test_data.USERS,
+        ])
+
+        normalized_data = copy.deepcopy(test_data.DEFAULT_DATA)
+        normalizer.normalize_default_data(normalized_data)
+
+    def tearDown(self):
+        super(TestDefaultDataProcessor, self).tearDown()
+
+    def test_normalizer(self):
+        data = copy.deepcopy(test_data.DEFAULT_DATA)
+
+        normalizer.normalize_default_data(data)
+
+        self.assertIn('releases', data['repos'][0])
+        self.assertEqual([], data['repos'][0]['releases'],
+                         message='Empty list of releases expected')
+        self.assertEqual(0, data['users'][0]['companies'][-1]['end_date'],
+                         message='The last company end date should be 0')
+        self.assertIn('user_id', data['users'][0])
+        self.assertEqual(test_data.USERS[0]['launchpad_id'],
+                         data['users'][0]['user_id'],
+                         message='User id should be set')
+
+        # verify that *independent company is added automatically
+        self.assertEqual(3, len(data['users'][1]['companies']))
+        self.assertEqual(0, data['users'][1]['companies'][-1]['end_date'],
+                         message='The last company end date should be 0')
+
+    def test_update_project_list(self):
+        with mock.patch('stackalytics.processor.default_data_processor.'
+                        '_retrieve_project_list_from_gerrit') as retriever:
+            retriever.return_value = [
+                {'module': 'nova',
+                 'uri': 'git://git.openstack.org/openstack/nova',
+                 'organization': 'openstack'},
+                {'module': 'qa', 'uri': 'git://git.openstack.org/openstack/qa',
+                 'organization': 'openstack'},
+            ]
+            dd = {
+                'repos': [
+                    {'module': 'qa',
+                     'uri': 'git://git.openstack.org/openstack/qa',
+                     'organization': 'openstack'},
+                    {'module': 'tux',
+                     'uri': 'git://git.openstack.org/stackforge/tux',
+                     'organization': 'stackforge'},
+                ],
+                'project_sources': [{'organization': 'openstack',
+                                     'uri': 'gerrit://'}],
+                'module_groups': [],
+            }
+
+            default_data_processor._update_project_list(dd)
+
+            self.assertEqual(3, len(dd['repos']))
+            self.assertIn('qa', set([r['module'] for r in dd['repos']]))
+            self.assertIn('nova', set([r['module'] for r in dd['repos']]))
+            self.assertIn('tux', set([r['module'] for r in dd['repos']]))
+
+            self.assertEqual(2, len(dd['module_groups']))
+            self.assertIn({'id': 'openstack',
+                           'module_group_name': 'openstack',
+                           'modules': ['qa', 'nova'],
+                           'tag': 'organization'}, dd['module_groups'])
+            self.assertIn({'id': 'stackforge',
+                           'module_group_name': 'stackforge',
+                           'modules': ['tux'],
+                           'tag': 'organization'}, dd['module_groups'])

stackalytics/tests/unit/test_driverlog.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import testtools
+
+from stackalytics.processor import driverlog
+
+
+class TestDriverlog(testtools.TestCase):
+    def setUp(self):
+        super(TestDriverlog, self).setUp()
+
+    def test_find_ci_result_voting_ci(self):
+        review = {
+            'record_type': 'review',
+            'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
+            'module': 'nova',
+            'branch': 'master',
+            'status': 'NEW',
+            'number': '97860',
+            'patchSets': [
+                {'number': '1',
+                 'approvals': [
+                     {'type': 'Verified', 'description': 'Verified',
+                      'value': '1', 'grantedOn': 1234567890 - 1,
+                      'by': {
+                          'name': 'Batman',
+                          'email': 'batman@openstack.org',
+                          'username': 'batman'}},
+                     {'type': 'Verified', 'description': 'Verified',
+                      'value': '-1', 'grantedOn': 1234567890,
+                      'by': {
+                          'name': 'Pikachu',
+                          'email': 'pikachu@openstack.org',
+                          'username': 'pikachu'}},
+                 ]}],
+            'comments': [
+                {'message': 'Patch Set 1: build successful',
+                 'reviewer': {'username': 'batman'},
+                 'timestamp': 1234567890}
+            ]}
+
+        ci_map = {
+            'batman': {
+                'name': 'Batman Driver',
+                'vendor': 'Gotham Inc',
+                'ci': {
+                    'id': 'batman'
+                }
+            }
+        }
+
+        res = list(driverlog.find_ci_result(review, ci_map))
+
+        expected_result = {
+            'reviewer': {'username': 'batman'},
+            'ci_result': True,
+            'is_merged': False,
+            'message': 'build successful',
+            'date': 1234567890,
+            'review_id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
+            'review_number': '97860',
+            'driver_name': 'Batman Driver',
+            'driver_vendor': 'Gotham Inc',
+        }
+        self.assertEqual(1, len(res), 'One CI result is expected')
+        self.assertEqual(expected_result, res[0])

stackalytics/tests/unit/test_dump.py

@@ -0,0 +1,68 @@
+# Copyright (c) 2014 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import memcache
+import mock
+import testtools
+
+from stackalytics.processor import dump
+
+
+class TestDump(testtools.TestCase):
+
+    def _make_data(self, record_count):
+        data = {'record:count': record_count}
+        for i in range(record_count):
+            data['record:%d' % i] = i
+        return data
+
+    def test_export_data_records(self):
+        record_count = 153
+        data = self._make_data(record_count)
+        memcache_inst = mock.Mock(memcache.Client)
+        memcache_inst.get = lambda x: data.get(x)
+        memcache_inst.get_multi = lambda keys, key_prefix: dict(
+            ('%s' % n, data.get(key_prefix + '%s' % n)) for n in keys)
+
+        with mock.patch('pickle.dump') as pickle_dump:
+            fd = mock.Mock()
+            dump.export_data(memcache_inst, fd)
+            # self.assertEquals(total, pickle_dump.call_count)
+
+            expected_calls = [mock.call(('record:count', record_count), fd)]
+            for i in range(record_count):
+                expected_calls.append(mock.call(('record:%d' % i,
+                                                 data['record:%d' % i]), fd))
+            pickle_dump.assert_has_calls(expected_calls, any_order=True)
+
+    def test_export_data_records_get_multi_truncates_chunk(self):
+        record_count = 153
+        data = self._make_data(record_count)
+        memcache_inst = mock.Mock(memcache.Client)
+        memcache_inst.get = lambda x: data.get(x)
+        memcache_inst.get_multi = lambda keys, key_prefix: dict(
+            ('%s' % n, data.get(key_prefix + '%s' % n))
+            for n in [k for k, v in zip(keys, range(len(keys) - 1))])
+
+        with mock.patch('pickle.dump') as pickle_dump:
+            fd = mock.Mock()
+            dump.export_data(memcache_inst, fd)
+            # self.assertEquals(total, pickle_dump.call_count)
+
+            expected_calls = [mock.call(('record:count', record_count), fd)]
+            for i in range(record_count):
+                expected_calls.append(mock.call(('record:%d' % i,
+                                                 data['record:%d' % i]), fd))
+            pickle_dump.assert_has_calls(expected_calls, any_order=True)

stackalytics/tests/unit/test_governance.py

@@ -0,0 +1,107 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import testtools
+
+from stackalytics.processor import governance
+
+
+SAMPLE = """
+Sahara:
+  ptl: Sergey Lukjanov (SergeyLukjanov)
+  irc-channel: openstack-sahara
+  service: Data processing service
+  mission: >
+    To provide a scalable data processing stack and associated management
+    interfaces.
+  url: https://wiki.openstack.org/wiki/Sahara
+  deliverables:
+    python-saharaclient:
+      repos:
+        - openstack/python-saharaclient
+      tags:
+        - release:cycle-with-intermediary
+        - release:has-stable-branches
+        - type:library
+        - release:managed
+        - vulnerability:managed
+    sahara:
+      repos:
+        - openstack/sahara
+        - openstack/sahara-extra
+        - openstack/sahara-image-elements
+      tags:
+        - tc-approved-release
+        - release:managed
+        - release:cycle-with-milestones
+        - release:has-stable-branches
+        - type:service
+        - vulnerability:managed
+    sahara-dashboard:
+      repos:
+        - openstack/sahara-dashboard
+      tags:
+        - type:library
+    sahara-specs:
+      repos:
+        - openstack/sahara-specs
+"""
+
+
+class TestGovernance(testtools.TestCase):
+
+    @mock.patch('stackalytics.processor.utils.read_uri')
+    def test_read_official_projects_yaml(self, read_uri):
+        read_uri.return_value = SAMPLE
+
+        expected = {
+            'sahara-group': {
+                'id': 'sahara-group',
+                'module_group_name': 'Sahara Official',
+                'modules': ['python-saharaclient', 'sahara',
+                            'sahara-dashboard', 'sahara-extra',
+                            'sahara-image-elements', 'sahara-specs'],
+                'tag': 'program'
+            },
+            'tc-approved-release': {
+                'id': 'tc-approved-release',
+                'module_group_name': 'tc-approved-release',
+                'modules': ['sahara', 'sahara-extra', 'sahara-image-elements'],
+                'tag': 'project_type'
+            },
+            'type:library': {
+                'id': 'type:library',
+                'module_group_name': 'type:library',
+                'modules': ['python-saharaclient', 'sahara-dashboard'],
+                'tag': 'project_type'
+            },
+            'type:service': {
+                'id': 'type:service',
+                'module_group_name': 'type:service',
+                'modules': ['sahara', 'sahara-extra', 'sahara-image-elements'],
+                'tag': 'project_type'
+            },
+            'openstack-official': {
+                'id': 'openstack-official',
+                'module_group_name': 'openstack-official',
+                'modules': ['python-saharaclient', 'sahara',
+                            'sahara-dashboard', 'sahara-extra',
+                            'sahara-image-elements', 'sahara-specs'],
+                'tag': 'project_type'
+            },
+        }
+
+        actual = governance.read_projects_yaml('uri')
+
+        self.assertEqual(expected, actual)

stackalytics/tests/unit/test_mls.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import testtools
+
+from stackalytics.processor import mls
+
+
+class TestMls(testtools.TestCase):
+    def setUp(self):
+        super(TestMls, self).setUp()
+
+    def test_mail_parse_regex(self):
+
+        content = '''
+URL: <http://lists.openstack.org/pipermail/openstack-dev/>
+
+From sorlando at nicira.com  Tue Jul 17 07:30:43 2012
+From: sorlando at nicira.com (Salvatore Orlando)
+Date: Tue, 17 Jul 2012 00:30:43 -0700
+Subject: [openstack-dev] [nova] [pci device passthrough] fails with
+ "NameError: global name '_' is not defined"
+In-Reply-To: <5004FBF1.1080102@redhat.com>
+References: <5004FBF1.1080102@redhat.com>
+Message-ID: <CAGR=i3htLvDOdh5u6mxqmo0zVP1eKKYAxAhj=e1-rpQWZOiF6Q@gmail.com>
+
+Good morning Gary!
+
+test works :)
+
+From sorlando at nicira.com  Tue Jul 17 07:30:43 2012
+From: sorlando at nicira.com (Salvatore Orlando)
+            '''
+        match = re.search(mls.MAIL_BOX_PATTERN, content)
+        self.assertTrue(match)
+        self.assertEqual('sorlando at nicira.com', match.group(1))
+        self.assertEqual('Salvatore Orlando', match.group(2))
+        self.assertEqual('Tue, 17 Jul 2012 00:30:43 -0700', match.group(3))
+        self.assertEqual('[openstack-dev] [nova] [pci device passthrough] '
+                         'fails with\n "NameError: global name \'_\' is not '
+                         'defined"', match.group(4))
+        self.assertEqual('<CAGR=i3htLvDOdh5u6mxqmo0zVP1eKKYAxAhj='
+                         'e1-rpQWZOiF6Q@gmail.com>', match.group(5))
+        self.assertEqual('Good morning Gary!\n\ntest works :)\n',
+                         match.group(6))

stackalytics/tests/unit/test_mps.py

@@ -0,0 +1,60 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import testtools
+
+from stackalytics.processor import mps
+
+
+class TestMps(testtools.TestCase):
+    def setUp(self):
+        super(TestMps, self).setUp()
+
+    def test_member_parse_regex(self):
+
+        content = '''<h1>Individual Member Profile</h1>
+<div class="candidate span-14">
+<div class="span-4">
+<img src="/themes/openstack/images/generic-profile-photo.png"><p>&nbsp;</p>
+</div>
+<a name="profile-10501"></a>
+<div class="details span-10 last">
+<div class="last name-and-title">
+<h3>Jim Battenberg</h3>
+</div>
+<hr><div class="span-4"><strong>Date Joined</strong></div>
+<div class="span-6 last">June 25, 2013 <br><br></div>
+    <div class="span-4"><strong>Affiliations</strong></div>
+    <div class="span-6 last">
+            <div>
+                <b>Rackspace</b> From  (Current)
+            </div>
+    </div>
+<div class="span-4"><strong>Statement of Interest </strong></div>
+<div class="span-6 last">
+<p>contribute logic and evangelize openstack</p>
+</div>
+<p>&nbsp;</p>'''
+
+        match = re.search(mps.NAME_AND_DATE_PATTERN, content)
+        self.assertTrue(match)
+        self.assertEqual('Jim Battenberg', match.group('member_name'))
+        self.assertEqual('June 25, 2013 ', match.group('date_joined'))
+
+        match = re.search(mps.COMPANY_PATTERN, content)
+        self.assertTrue(match)
+        self.assertEqual('Rackspace', match.group('company_draft'))

stackalytics/tests/unit/test_user_processor.py

@@ -0,0 +1,89 @@
+# Copyright (c) 2015 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import testtools
+
+from stackalytics.processor import user_processor
+
+
+class TestUserProcessor(testtools.TestCase):
+    def setUp(self):
+        super(TestUserProcessor, self).setUp()
+
+    def test_update_user(self):
+        user = {
+            "launchpad_id": "user",
+            "companies": [
+                {
+                    "company_name": "Rackspace",
+                    "end_date": "2011-Nov-20"
+                },
+                {
+                    "company_name": "IBM",
+                    "end_date": None
+                }
+            ],
+            "user_name": "John Smith",
+            "emails": ["john@smith.com"]
+        }
+
+        stored_user = {
+            "launchpad_id": "user",
+            "companies": [
+                {
+                    "company_name": "Rackspace",
+                    "end_date": "2011-Nov-20"
+                },
+                {
+                    "company_name": "IBM",
+                    "end_date": None
+                }
+            ],
+            "user_name": "Johnny",
+            "emails": ["john@smith.com", "mapped_email@gmail.com"],
+            "static": True
+        }
+
+        updated_user = user_processor.update_user_profile(stored_user, user)
+
+        # merge emails from profile with those discovered by Stackalytics
+        self.assertEqual(set(stored_user['emails']),
+                         set(updated_user['emails']))
+        # name from the profile has higher priority over mined
+        self.assertEqual(user['user_name'], updated_user['user_name'])
+        # static flag must present
+        self.assertTrue(updated_user.get('static'))
+
+    def test_update_user_unknown_user(self):
+        user = {
+            "launchpad_id": "user",
+            "companies": [
+                {
+                    "company_name": "Rackspace",
+                    "end_date": "2011-Nov-20"
+                },
+                {
+                    "company_name": "IBM",
+                    "end_date": None
+                }
+            ],
+            "user_name": "John Smith",
+            "emails": ["john@smith.com"]
+        }
+
+        stored_user = None
+
+        updated_user = user_processor.update_user_profile(stored_user, user)
+        self.assertTrue(updated_user.get('static'))

stackalytics/tests/unit/test_utils.py

@@ -0,0 +1,125 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import testtools
+
+from stackalytics.processor import utils
+
+
+class TestUtils(testtools.TestCase):
+    def setUp(self):
+        super(TestUtils, self).setUp()
+
+    def _test_one_range(self, start, end, step):
+        elements = set()
+        for chunk in utils.make_range(start, end, step):
+            for item in chunk:
+                self.assertFalse(item in elements)
+                elements.add(item)
+
+        self.assertTrue(set(range(start, end)) == elements)
+
+    def test_make_range_0_10_1(self):
+        self._test_one_range(0, 10, 1)
+
+    def test_make_range_0_10_3(self):
+        self._test_one_range(0, 10, 3)
+
+    def test_make_range_3_5_4(self):
+        self._test_one_range(3, 5, 4)
+
+    def test_make_range_5_26_10(self):
+        self._test_one_range(5, 26, 10)
+
+    def test_email_valid(self):
+        self.assertTrue(utils.check_email_validity('pupkin@gmail.com'))
+        self.assertTrue(utils.check_email_validity('v.pup_kin2@ntt.co.jp'))
+
+    def test_email_invalid(self):
+        self.assertFalse(utils.check_email_validity('pupkin@localhost'))
+        self.assertFalse(utils.check_email_validity('222@some.(trash)'))
+
+    def test_unwrap(self):
+        original = 'Lorem ipsum. Dolor\nsit amet.\n Lorem\n ipsum.\ndolor!\n'
+        expected = 'Lorem ipsum. Dolor sit amet.\n Lorem\n ipsum.\ndolor!'
+
+        self.assertEqual(expected, utils.unwrap_text(original))
+
+    def test_format_text_split_long_link(self):
+        original = ('https://blueprints.launchpad.net/stackalytics/+spec/'
+                    'stackalytics-core')
+        expected = ('https://blueprints.​launchpad.​net'
+                    '/​stackalytics/+spec/​stackalytics-core')
+
+        self.assertEqual(expected, utils.format_text(original))
+
+    def test_format_text_split_full_class_path(self):
+        original = 'tests.unit.benchmark.scenarios.test_base'
+        expected = ('tests.​unit.​benchmark.​'
+                    'scenarios.​test_base')
+
+        self.assertEqual(expected, utils.format_text(original))
+
+    def test_format_text_split_full_class_path_middle_line(self):
+        original = 'some text tests.unit.benchmark.scenarios.test_base wide'
+        expected = ('some text tests.​unit.​benchmark.​'
+                    'scenarios.​test_base wide')
+
+        self.assertEqual(expected, utils.format_text(original))
+
+    def test_add_index(self):
+        sequence = [{'name': 'A'}, {'name': 'B'}, {'name': 'C'}]
+        expected = [{'index': 1, 'name': 'A'}, {'index': 2, 'name': 'B'},
+                    {'index': 3, 'name': 'C'}]
+        self.assertEqual(expected, utils.add_index(sequence))
+
+    def test_add_index_with_filter(self):
+        sequence = [{'name': 'A'}, {'name': 'B'}, {'name': 'C'}]
+        expected = [{'index': 0, 'name': 'A'}, {'index': '', 'name': 'B'},
+                    {'index': 1, 'name': 'C'}]
+        self.assertEqual(expected, utils.add_index(
+            sequence, start=0, item_filter=lambda x: x['name'] != 'B'))
+
+    def test_keep_safe_chars(self):
+        self.assertEqual('somemoretext',
+                         utils.keep_safe_chars('some more text'))
+        self.assertEqual(u'(unicode)',
+                         utils.keep_safe_chars(u'(unicode \u0423) '))
+
+    def test_normalize_company_name(self):
+        company_names = ['EMC Corporation', 'Abc, corp..', 'Mirantis IT.',
+                         'Red Hat, Inc.', 'abc s.r.o. ABC', '2s.r.o. co',
+                         'AL.P.B L.P. s.r.o. s.r.o. C ltd.']
+        correct_normalized_company_names = ['emc', 'abc', 'mirantis',
+                                            'redhat', 'abcabc', '2sro',
+                                            'alpbc']
+        normalized_company_names = [utils.normalize_company_name(name)
+                                    for name in company_names]
+
+        self.assertEqual(normalized_company_names,
+                         correct_normalized_company_names)
+
+    def test_validate_lp_display_name(self):
+        profile = dict(name='johnny', display_name='John Smith')
+        utils.validate_lp_display_name(profile)
+        self.assertEqual('John Smith', profile['display_name'])
+
+        profile = dict(name='johnny', display_name='<email address hidden>')
+        utils.validate_lp_display_name(profile)
+        self.assertEqual('johnny', profile['display_name'])
+
+        profile = None
+        utils.validate_lp_display_name(profile)
+        self.assertEqual(None, profile)

stackalytics/tests/unit/test_vcs.py

@@ -0,0 +1,219 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import testtools
+
+from stackalytics.processor import vcs
+
+
+class TestVcsProcessor(testtools.TestCase):
+    def setUp(self):
+        super(TestVcsProcessor, self).setUp()
+
+        self.repo = {
+            'module': 'dummy',
+            'uri': 'git://git.openstack.org/dummy.git',
+            'releases': []
+        }
+        self.git = vcs.Git(self.repo, '/tmp')
+        self.chdir_patcher = mock.patch('os.chdir')
+        self.chdir_patcher.start()
+
+    def tearDown(self):
+        super(TestVcsProcessor, self).tearDown()
+        self.chdir_patcher.stop()
+
+    def test_git_log(self):
+        with mock.patch('sh.git') as git_mock:
+            git_mock.return_value = '''
+commit_id:b5a416ac344160512f95751ae16e6612aefd4a57
+date:1369119386
+author_name:Akihiro MOTOKI
+author_email:motoki@da.jp.nec.com
+subject:Remove class-based import in the code repo
+message:Fixes bug 1167901.
+
+This commit also removes backslashes for line break.
+
+Change-Id: Id26fdfd2af4862652d7270aec132d40662efeb96
+
+diff_stat:
+
+ 21 files changed, 340 insertions(+), 408 deletions(-)
+commit_id:5be031f81f76d68c6e4cbaad2247044aca179843
+date:1370975889
+author_name:Monty Taylor
+author_email:mordred@inaugust.com
+subject:Remove explicit distribute depend.
+message:Causes issues with the recent re-merge with setuptools. Advice from
+upstream is to stop doing explicit depends.
+
+Change-Id: I70638f239794e78ba049c60d2001190910a89c90
+
+diff_stat:
+
+ 1 file changed, 1 deletion(-)
+commit_id:2dcb4fa4aa1925ffbd90d1cc7556a13a1bc45d1c
+date:1369831203
+author_name:Mark McClain
+author_email:mark.mcclain@dreamhost.com
+subject:add readme for 2.2.2
+message:Fixes bug: 1234567
+Also fixes bug 987654
+Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
+
+diff_stat:
+
+ 1 file changed, 8 insertions(+)
+commit_id:06d321b6b7681b162cd3231b5bdd92b17eb4f401
+date:1369831203
+author_name:John Doe
+author_email:john.doe@dreamhost.com
+subject:add readme for 2.2.2
+message: implements blueprint fix-me.
+Co-Authored-By: Anonymous <wrong@email>
+Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
+
+diff_stat:
+
+ 0 files changed
+commit_id:913c86a9d5b6a1b74db36266e996cb4d6073f75b
+date:1369831203
+author_name:Doug Hoffner
+author_email:mark.mcclain@dreamhost.com
+subject:add readme for 2.2.2
+message:Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
+Co-Authored-By: some friend of mine
+
+diff_stat:
+
+ 0 files changed, 0 insertions(+), 0 deletions(-)
+commit_id:2f3103a96c4d234a4fcc0b0211a20308c0d342e7
+date:1397687866
+author_name:James E. Blair
+author_email:jeblair@openstack.org
+subject:Reduce IAD usage by 50%
+message:At provider's request.
+
+Change-Id: I976eaff357bf0ad4bce2a7fd5fe6fd81750276c5
+
+diff_stat:
+commit_id:12811c76f3a8208b36f81e61451ec17d227b4e58
+date:1369831203
+author_name:Jimi Hendrix
+author_email:jimi.hendrix@openstack.com
+subject:adds support off co-authors
+message:Change-Id: Id811c762ec1d13992b306c4a38e7360575e61451
+Co-Authored-By: Tupac Shakur <tupac.shakur@openstack.com>
+Also-By: Bob Dylan <bob.dylan@openstack.com>
+Also-By: Anonymous <wrong@email>
+Also-By: Winnie the Pooh winnie222@openstack.org
+
+diff_stat:
+
+ 0 files changed, 0 insertions(+), 0 deletions(-)
+commit_id:d1af9cbe0187e1a65cf1eb46fb1650cf619a7b3a
+date:1369831300
+author_name:Vasya Pupkin
+author_email:vpupkinx@openstack.com
+subject:adds new support of co-authors
+message:Change-Id: I577dfdf7f65a0c883ddbcfda62daf8c5f9c746c1
+Co-Authored-By: Tupac Shakur <tupac.shakur@openstack.com>
+Also: Bob Dylan <bob.dylan@openstack.com>
+Co-Authored: Anonymous <correct@email.com>
+Co-Author-By: Anonymous2 <correct@email2.com>
+Co-Author: Winnie the Pooh winnie222@openstack.org
+
+diff_stat:
+
+ 0 files changed, 0 insertions(+), 0 deletions(-)
+            '''
+            commits = list(self.git.log('dummy', 'dummy'))
+
+        commits_expected = 8
+        self.assertEqual(commits_expected, len(commits))
+
+        self.assertEqual(21, commits[0]['files_changed'])
+        self.assertEqual(340, commits[0]['lines_added'])
+        self.assertEqual(408, commits[0]['lines_deleted'])
+        self.assertEqual(['1167901'], commits[0]['bug_id'])
+
+        self.assertEqual(1, commits[1]['files_changed'])
+        self.assertEqual(0, commits[1]['lines_added'])
+        self.assertEqual(1, commits[1]['lines_deleted'])
+
+        self.assertEqual(1, commits[2]['files_changed'])
+        self.assertEqual(8, commits[2]['lines_added'])
+        self.assertEqual(0, commits[2]['lines_deleted'])
+        self.assertEqual(set(['987654', '1234567']),
+                         set(commits[2]['bug_id']))
+
+        self.assertEqual(0, commits[3]['files_changed'])
+        self.assertEqual(0, commits[3]['lines_added'])
+        self.assertEqual(0, commits[3]['lines_deleted'])
+        self.assertEqual(set(['dummy:fix-me']),
+                         set(commits[3]['blueprint_id']))
+        self.assertFalse('coauthor' in commits[3])
+
+        self.assertEqual(0, commits[4]['files_changed'])
+        self.assertEqual(0, commits[4]['lines_added'])
+        self.assertEqual(0, commits[4]['lines_deleted'])
+        self.assertFalse('coauthor' in commits[4])
+
+        self.assertEqual('jeblair@openstack.org', commits[5]['author_email'])
+        self.assertEqual(0, commits[5]['files_changed'])
+        self.assertEqual(0, commits[5]['lines_added'])
+        self.assertEqual(0, commits[5]['lines_deleted'])
+
+        self.assertIn(
+            {'author_name': 'Tupac Shakur',
+             'author_email': 'tupac.shakur@openstack.com'},
+            commits[6]['coauthor'])
+
+        self.assertIn(
+            {'author_name': 'Bob Dylan',
+             'author_email': 'bob.dylan@openstack.com'},
+            commits[6]['coauthor'])
+
+        self.assertIn(
+            {'author_name': 'Winnie the Pooh',
+             'author_email': 'winnie222@openstack.org'},
+            commits[6]['coauthor'])
+
+        self.assertIn(
+            {'author_name': 'Tupac Shakur',
+             'author_email': 'tupac.shakur@openstack.com'},
+            commits[7]['coauthor'])
+
+        self.assertNotIn(
+            {'author_name': 'Bob Dylan',
+             'author_email': 'bob.dylan@openstack.com'},
+            commits[7]['coauthor'])
+
+        self.assertNotIn(
+            {'author_name': 'Anonymous',
+             'author_email': 'correct@email.com'},
+            commits[7]['coauthor'])
+
+        self.assertNotIn(
+            {'author_name': 'Anonymous2',
+             'author_email': 'correct@email2.com'},
+            commits[7]['coauthor'])
+
+        self.assertIn(
+            {'author_name': 'Winnie the Pooh',
+             'author_email': 'winnie222@openstack.org'},
+            commits[7]['coauthor'])

stackalytics/tests/unit/test_web_utils.py

@@ -0,0 +1,161 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import testtools
+
+from stackalytics.dashboard import helpers
+from stackalytics.dashboard import parameters
+
+
+class TestWebUtils(testtools.TestCase):
+    def setUp(self):
+        super(TestWebUtils, self).setUp()
+
+    def test_make_commit_message(self):
+        message = '''
+During finish_migration the manager calls initialize_connection but doesn't
+update the block_device_mapping with the potentially new connection_info
+returned.
+
+
+Fixes bug 1076801
+Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db'''
+
+        module = 'test'
+
+        record = {
+            'message': message,
+            'module': module,
+        }
+
+        expected = '''\
+During finish_migration the manager calls initialize_connection but doesn't \
+update the block_device_mapping with the potentially new connection_info \
+returned.
+Fixes bug <a href="https://bugs.launchpad.net/bugs/1076801" class="ext_link">\
+1076801</a>
+''' + (
+            'Change-Id: <a href="https://review.openstack.org/#q,'
+            'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">'
+            'Ie49ccd2138905e178843b375a9b16c3fe572d1db</a>')
+
+        observed = helpers.make_commit_message(record)
+
+        self.assertEqual(expected, observed,
+                         'Commit message should be processed correctly')
+
+    def test_make_commit_message_blueprint_link(self):
+        message = '''
+Implemented new driver for Cinder <:
+Implements Blueprint super-driver
+Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db'''
+
+        module = 'cinder'
+
+        record = {
+            'message': message,
+            'module': module,
+        }
+
+        expected = '''\
+Implemented new driver for Cinder &lt;:
+Implements Blueprint ''' + (
+            '<a href="https://blueprints.launchpad.net/cinder/+spec/'
+            'super-driver" class="ext_link">super-driver</a>' + '\n' +
+            'Change-Id: <a href="https://review.openstack.org/#q,'
+            'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">'
+            'Ie49ccd2138905e178843b375a9b16c3fe572d1db</a>')
+
+        observed = helpers.make_commit_message(record)
+
+        self.assertEqual(expected, observed,
+                         'Commit message should be processed correctly')
+
+    @mock.patch('stackalytics.dashboard.vault.get_vault')
+    @mock.patch('stackalytics.dashboard.vault.get_user_from_runtime_storage')
+    def test_make_page_title(self, user_patch, vault_patch):
+        def _pt(id, title=None):
+            return dict(id=id.lower(), title=title or id)
+
+        user_inst = {'user_name': 'John Doe'}
+        module_inst = {'module_group_name': 'neutron'}
+
+        self.assertEqual('OpenStack community contribution in all releases',
+                         helpers.make_page_title(
+                             _pt('OpenStack'), 'all', None, None, None))
+        self.assertEqual('OpenStack community contribution in Havana release',
+                         helpers.make_page_title(
+                             _pt('OpenStack'), 'Havana', None, None, None))
+        self.assertEqual('Mirantis contribution in Havana release',
+                         helpers.make_page_title(
+                             _pt('Stackforge'), 'Havana', None, 'Mirantis',
+                             None))
+        self.assertEqual('John Doe contribution in Havana release',
+                         helpers.make_page_title(
+                             _pt('all'), 'Havana', None, None, user_inst))
+        self.assertEqual(
+            'John Doe (Mirantis) contribution to neutron in Havana release',
+            helpers.make_page_title(
+                _pt('all'), 'Havana', module_inst, 'Mirantis', user_inst))
+        self.assertEqual('Ansible community contribution during OpenStack '
+                         'Havana release',
+                         helpers.make_page_title(
+                             _pt('Ansible'), 'Havana', None, None, None))
+        self.assertEqual('Docker community contribution',
+                         helpers.make_page_title(
+                             _pt('Docker'), 'all', None, None, None))
+
+    @mock.patch('flask.request')
+    @mock.patch('stackalytics.dashboard.parameters.get_default')
+    def test_parameters_get_parameter(self, get_default, flask_request):
+
+        flask_request.args = mock.Mock()
+        flask_request.args.get = mock.Mock(side_effect=lambda x: x)
+
+        def make(values=None):
+            def f(arg):
+                return values.get(arg, None) if values else None
+            return f
+
+        get_default.side_effect = make()
+        flask_request.args.get.side_effect = make({'param': 'foo'})
+        self.assertEqual(['foo'], parameters.get_parameter(
+            {'param': 'foo'}, 'param'))
+
+        flask_request.args.get.side_effect = make({'param': 'foo'})
+        self.assertEqual(['foo'], parameters.get_parameter({}, 'param'))
+
+        flask_request.args.get.side_effect = make({'param': 'foo'})
+        self.assertEqual([], parameters.get_parameter(
+            {}, 'other', use_default=False))
+
+        flask_request.args.get.side_effect = make({'params': 'foo'})
+        self.assertEqual(['foo'], parameters.get_parameter(
+            {}, 'param', plural_name='params'))
+
+        flask_request.args.get.side_effect = make({})
+        get_default.side_effect = make({'param': 'foo'})
+        self.assertEqual(['foo'], parameters.get_parameter({}, 'param'))
+        self.assertEqual([], parameters.get_parameter({}, 'other'))
+
+    def test_filter_bug_title(self):
+        bug_title = ('Bug #1459454 in Barbican: "Stored key certificate '
+                     'order does not set PK on generated container"')
+        expected = ('Stored key certificate order does not set PK '
+                    'on generated container')
+
+        actual = helpers.filter_bug_title(bug_title)
+        self.assertEqual(expected, actual)