Merge "Allow to exclude module name from launchpad query"

stackalytics/processor/bps.py

@@ -70,7 +70,9 @@ def _log_module(module, primary_module, modified_since):
 
 def log(repo, modified_since):
     repo_module = repo['module']
-    modules = [repo_module] + repo.get('aliases', [])
+    modules = repo.get('aliases', [])
+    if repo['launchpad_name']:
+        modules.append(repo['launchpad_name'])
 
     for module in modules:
         if not launchpad_utils.lp_module_exists(module):

stackalytics/processor/default_data_processor.py

@@ -85,9 +85,11 @@ def _retrieve_project_list_from_gerrit(project_source):
         raise
 
     git_base_uri = project_source.get('git_base_uri') or CONF.git_base_uri
+    use_launchpad_metrics = project_source.get('launchpad_metrics', True)
 
     for repo in git_repos:
         name = repo.split('/')[-1]
+        launchpad_name = name if use_launchpad_metrics else None
         repo_uri = '%(git_base_uri)s/%(repo)s.git' % dict(
             git_base_uri=git_base_uri, repo=repo)
         yield {
@@ -100,6 +102,7 @@ def _retrieve_project_list_from_gerrit(project_source):
             'gerrit_uri': uri,
             'ssh_username': username,
             'key_filename': key_filename,
+            'launchpad_name': launchpad_name,
         }
 
 
@@ -117,14 +120,18 @@ def _retrieve_project_list_from_github(project_source):
         LOG.error('Failed to retrieve list of projects from GitHub: %s',
                   e, exc_info=True)
         raise
+    use_launchpad_metrics = project_source.get('launchpad_metrics', True)
 
     for repo in github_repos:
+        name = repo.name.lower()
+        launchpad_name = name if use_launchpad_metrics else None
         yield {
             'branches': [project_source.get('default_branch', 'master')],
-            'module': repo.name.lower(),
+            'module': name,
             'organization': organization,
             'uri': repo.git_url,
-            'releases': []
+            'releases': [],
+            'launchpad_name': launchpad_name,
         }
 
 

stackalytics/processor/lp.py

@@ -47,7 +47,9 @@ def _log_module(module, primary_module):
 
 def log(repo):
     repo_module = repo['module']
-    modules = [repo_module] + repo.get('aliases', [])
+    modules = repo.get('aliases', [])
+    if repo['launchpad_name']:
+        modules.append(repo['launchpad_name'])
 
     for module in modules:
         if not launchpad_utils.lp_module_exists(module):

stackalytics/processor/main.py

@@ -182,9 +182,12 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst):
 
     _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst)
 
+    if repo['launchpad_name'] or repo.get('aliases'):
         _process_repo_bugs(repo, runtime_storage_inst, record_processor_inst)
-
+        _process_repo_blueprints(repo, runtime_storage_inst,
-    _process_repo_blueprints(repo, runtime_storage_inst, record_processor_inst)
+                                 record_processor_inst)
+    else:
+        LOG.info('Skipping launchpad data for repo: %s', repo['uri'])
 
     if 'gerrit_uri' in repo:
         _process_repo_reviews(repo, runtime_storage_inst,

stackalytics/processor/schema.py

@@ -104,6 +104,9 @@ default_data = {
                     "module": {
                         "type": "string"
                     },
+                    "launchpad_name": {
+                        "type": ["string", "null"]
+                    },
                     "releases": {
                         "type": "array",
                         "items": {
@@ -197,6 +200,9 @@ default_data = {
                     "module_group_id": {
                         "type": "string"
                     },
+                    "launchpad_metrics": {
+                        "type": "boolean"
+                    },
                     "pattern": {
                         "type": "string"
                     }

stackalytics/tests/unit/test_bps.py

@@ -165,7 +165,8 @@ class TestBps(testtools.TestCase):
     @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
     def test_log(self, lp_bug_generator):
         repo = {
-            'module': 'sahara'
+            'module': 'sahara',
+            'launchpad_name': 'sahara',
         }
         modified_since = 1234567890
         lp_bug_generator.return_value = iter([BUG])
@@ -191,7 +192,8 @@ class TestBps(testtools.TestCase):
     @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
     def test_log_released_not_committed(self, lp_bug_generator):
         repo = {
-            'module': 'sahara'
+            'module': 'sahara',
+            'launchpad_name': 'sahara',
         }
         modified_since = 1234567890
         lp_bug_generator.return_value = iter([RELEASED_NOT_COMMITTED_BUG])
@@ -217,7 +219,8 @@ class TestBps(testtools.TestCase):
     def test_log_additional_module(self, lp_bug_generator):
         # bug linked to another project should not appear
         repo = {
-            'module': 'sahara'
+            'module': 'sahara',
+            'launchpad_name': 'sahara',
         }
         modified_since = 1234567890
         lp_bug_generator.return_value = iter([BUG, LINKED_BUG])
@@ -244,7 +247,8 @@ class TestBps(testtools.TestCase):
     def test_log_additional_milestone(self, lp_bug_generator):
         # bug linked to different milestone should be mapped to the release
         repo = {
-            'module': 'sahara'
+            'module': 'sahara',
+            'launchpad_name': 'sahara',
         }
         modified_since = 1234567890
         lp_bug_generator.return_value = iter([BUG, ANOTHER_MILESTONE_BUG])
@@ -288,11 +292,76 @@ class TestBps(testtools.TestCase):
         # bug linked to another project should not appear
         repo = {
             'module': 'savanna',
+            'launchpad_name': None,
             'aliases': ['sahara']
         }
         modified_since = 1234567890
         lp_bug_generator.return_value = iter([BUG])
-        lp_module_exists.side_effect = iter([False, True])
+        lp_module_exists.side_effect = iter([True])
+
+        expected = [{
+            'assignee': 'slukjanov',
+            'date_created': 1433252154,
+            'date_fix_committed': 1433266265,
+            'date_fix_released': 1433266265,
+            'id': 'savanna/1458945',
+            'importance': 'Medium',
+            'module': 'savanna',  # should be the same as primary module name
+            'owner': 'samueldmq',
+            'status': 'Fix Released',
+            'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"',
+            'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945'
+        }]
+
+        actual = list(bps.log(repo, modified_since))
+
+        self.assertEqual(expected, actual)
+
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists')
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
+    def test_log_module_launchpad_name_none(self, lp_bug_generator,
+                                            lp_module_exists):
+        repo = {
+            'module': 'savanna',
+            'launchpad_name': None,
+        }
+        modified_since = 1234567890
+        lp_bug_generator.return_value = iter([BUG])
+        lp_module_exists.side_effect = iter([True])
+
+        expected = []
+
+        actual = list(bps.log(repo, modified_since))
+
+        lp_module_exists.assert_not_called()
+        self.assertEqual(expected, actual)
+
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists')
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
+    def test_log_module_launchpad_name_with_alias(self, lp_bug_generator,
+                                                  lp_module_exists):
+        repo = {
+            'module': 'savanna',
+            'launchpad_name': 'sahara',
+            'aliases': ['someothername'],
+        }
+        modified_since = 1234567890
+        list(bps.log(repo, modified_since))
+
+        assert lp_module_exists.asser_has_calls(
+            [mock.call('sahara'), mock.call('someothername')], any_order=True)
+
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists')
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
+    def test_log_module_launchpad_name(self, lp_bug_generator,
+                                       lp_module_exists):
+        repo = {
+            'module': 'savanna',
+            'launchpad_name': 'sahara',
+        }
+        modified_since = 1234567890
+        lp_bug_generator.return_value = iter([BUG])
+        lp_module_exists.side_effect = iter([True])
 
         expected = [{
             'assignee': 'slukjanov',