Merge "Poll gerrit only for projects that have it"

stackalytics/processor/default_data_processor.py

@@ -94,7 +94,8 @@ def _retrieve_project_list_from_gerrit(project_source):
             'module': name,
             'organization': org,
             'uri': repo_uri,
-            'releases': []
+            'releases': [],
+            'has_gerrit': True,
         }
 
 
@@ -147,7 +148,14 @@ def _update_project_list(default_data):
     repos = _retrieve_project_list_from_sources(
         default_data['project_sources'])
     if repos:
-        default_data['repos'] += [r for r in repos
+        # update pre-configured
+        repos_dict = dict((r['uri'], r) for r in repos)
+        for r in default_data['repos']:
+            if r['uri'] in repos_dict:
+                r.update(repos_dict[r['uri']])
+
+        # update default data
+        default_data['repos'] += [r for r in repos_dict.values()
                                   if r['uri'] not in configured_repos]
 
     default_data['module_groups'] += _create_module_groups_for_project_sources(

stackalytics/processor/main.py

@@ -131,6 +131,9 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst,
         last_id = vcs_inst.get_last_id(branch)
         runtime_storage_inst.set_by_key(vcs_key, last_id)
 
+        if 'has_gerrit' not in repo:
+            continue  # do not poll reviews for those that do not have them
+
         LOG.info('Processing reviews for repo: %s, branch: %s', uri, branch)
 
         rcs_key = 'rcs:%s:%s' % (quoted_uri, branch)

stackalytics/tests/unit/test_default_data_processor.py

@@ -65,6 +65,7 @@ class TestDefaultDataProcessor(testtools.TestCase):
                  'uri': 'git://git.openstack.org/openstack/nova',
                  'organization': 'openstack'},
                 {'module': 'qa', 'uri': 'git://git.openstack.org/openstack/qa',
+                 'has_gerrit': True,
                  'organization': 'openstack'},
             ]
             dd = {
@@ -88,6 +89,10 @@ class TestDefaultDataProcessor(testtools.TestCase):
             self.assertIn('nova', set([r['module'] for r in dd['repos']]))
             self.assertIn('tux', set([r['module'] for r in dd['repos']]))
 
+            self.assertIn('has_gerrit', dd['repos'][0])
+            self.assertNotIn('has_gerrit', dd['repos'][1])
+            self.assertNotIn('has_gerrit', dd['repos'][2])
+
             self.assertEqual(2, len(dd['module_groups']))
             self.assertIn({'id': 'openstack',
                            'module_group_name': 'openstack',