diff --git a/stackalytics/processor/bps.py b/stackalytics/processor/bps.py index f6c07adcf..4c93ff798 100644 --- a/stackalytics/processor/bps.py +++ b/stackalytics/processor/bps.py @@ -70,7 +70,9 @@ def _log_module(module, primary_module, modified_since): def log(repo, modified_since): repo_module = repo['module'] - modules = [repo_module] + repo.get('aliases', []) + modules = repo.get('aliases', []) + if repo['launchpad_name']: + modules.append(repo['launchpad_name']) for module in modules: if not launchpad_utils.lp_module_exists(module): diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py index 5cf59f20b..02521ffbe 100644 --- a/stackalytics/processor/default_data_processor.py +++ b/stackalytics/processor/default_data_processor.py @@ -85,9 +85,11 @@ def _retrieve_project_list_from_gerrit(project_source): raise git_base_uri = project_source.get('git_base_uri') or CONF.git_base_uri + use_launchpad_metrics = project_source.get('launchpad_metrics', True) for repo in git_repos: name = repo.split('/')[-1] + launchpad_name = name if use_launchpad_metrics else None repo_uri = '%(git_base_uri)s/%(repo)s.git' % dict( git_base_uri=git_base_uri, repo=repo) yield { @@ -100,6 +102,7 @@ def _retrieve_project_list_from_gerrit(project_source): 'gerrit_uri': uri, 'ssh_username': username, 'key_filename': key_filename, + 'launchpad_name': launchpad_name, } @@ -115,14 +118,18 @@ def _retrieve_project_list_from_github(project_source): LOG.error('Failed to retrieve list of projects from GitHub: %s', e, exc_info=True) raise + use_launchpad_metrics = project_source.get('launchpad_metrics', True) for repo in github_repos: + name = repo.name.lower() + launchpad_name = name if use_launchpad_metrics else None yield { 'branches': [project_source.get('default_branch', 'master')], - 'module': repo.name.lower(), + 'module': name, 'organization': organization, 'uri': repo.git_url, - 'releases': [] + 'releases': [], + 'launchpad_name': launchpad_name, } diff --git a/stackalytics/processor/lp.py b/stackalytics/processor/lp.py index f35f6f52e..186edbb2d 100644 --- a/stackalytics/processor/lp.py +++ b/stackalytics/processor/lp.py @@ -47,7 +47,9 @@ def _log_module(module, primary_module): def log(repo): repo_module = repo['module'] - modules = [repo_module] + repo.get('aliases', []) + modules = repo.get('aliases', []) + if repo['launchpad_name']: + modules.append(repo['launchpad_name']) for module in modules: if not launchpad_utils.lp_module_exists(module): diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 498817eff..b120e0ebe 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -182,9 +182,12 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst): _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst) - _process_repo_bugs(repo, runtime_storage_inst, record_processor_inst) - - _process_repo_blueprints(repo, runtime_storage_inst, record_processor_inst) + if repo['launchpad_name'] or repo.get('aliases'): + _process_repo_bugs(repo, runtime_storage_inst, record_processor_inst) + _process_repo_blueprints(repo, runtime_storage_inst, + record_processor_inst) + else: + LOG.info('Skipping launchpad data for repo: %s', repo['uri']) if 'gerrit_uri' in repo: _process_repo_reviews(repo, runtime_storage_inst, diff --git a/stackalytics/processor/schema.py b/stackalytics/processor/schema.py index 65b220c66..3c79f8611 100644 --- a/stackalytics/processor/schema.py +++ b/stackalytics/processor/schema.py @@ -101,6 +101,9 @@ default_data = { "module": { "type": "string" }, + "launchpad_name": { + "type": ["string", "null"] + }, "releases": { "type": "array", "items": { @@ -194,6 +197,9 @@ default_data = { "module_group_id": { "type": "string" }, + "launchpad_metrics": { + "type": "boolean" + }, "pattern": { "type": "string" } diff --git a/stackalytics/tests/unit/test_bps.py b/stackalytics/tests/unit/test_bps.py index df58586a2..f581f59b3 100644 --- a/stackalytics/tests/unit/test_bps.py +++ b/stackalytics/tests/unit/test_bps.py @@ -165,7 +165,8 @@ class TestBps(testtools.TestCase): @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') def test_log(self, lp_bug_generator): repo = { - 'module': 'sahara' + 'module': 'sahara', + 'launchpad_name': 'sahara', } modified_since = 1234567890 lp_bug_generator.return_value = iter([BUG]) @@ -191,7 +192,8 @@ class TestBps(testtools.TestCase): @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') def test_log_released_not_committed(self, lp_bug_generator): repo = { - 'module': 'sahara' + 'module': 'sahara', + 'launchpad_name': 'sahara', } modified_since = 1234567890 lp_bug_generator.return_value = iter([RELEASED_NOT_COMMITTED_BUG]) @@ -217,7 +219,8 @@ class TestBps(testtools.TestCase): def test_log_additional_module(self, lp_bug_generator): # bug linked to another project should not appear repo = { - 'module': 'sahara' + 'module': 'sahara', + 'launchpad_name': 'sahara', } modified_since = 1234567890 lp_bug_generator.return_value = iter([BUG, LINKED_BUG]) @@ -244,7 +247,8 @@ class TestBps(testtools.TestCase): def test_log_additional_milestone(self, lp_bug_generator): # bug linked to different milestone should be mapped to the release repo = { - 'module': 'sahara' + 'module': 'sahara', + 'launchpad_name': 'sahara', } modified_since = 1234567890 lp_bug_generator.return_value = iter([BUG, ANOTHER_MILESTONE_BUG]) @@ -288,11 +292,76 @@ class TestBps(testtools.TestCase): # bug linked to another project should not appear repo = { 'module': 'savanna', + 'launchpad_name': None, 'aliases': ['sahara'] } modified_since = 1234567890 lp_bug_generator.return_value = iter([BUG]) - lp_module_exists.side_effect = iter([False, True]) + lp_module_exists.side_effect = iter([True]) + + expected = [{ + 'assignee': 'slukjanov', + 'date_created': 1433252154, + 'date_fix_committed': 1433266265, + 'date_fix_released': 1433266265, + 'id': 'savanna/1458945', + 'importance': 'Medium', + 'module': 'savanna', # should be the same as primary module name + 'owner': 'samueldmq', + 'status': 'Fix Released', + 'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"', + 'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945' + }] + + actual = list(bps.log(repo, modified_since)) + + self.assertEqual(expected, actual) + + @mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists') + @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') + def test_log_module_launchpad_name_none(self, lp_bug_generator, + lp_module_exists): + repo = { + 'module': 'savanna', + 'launchpad_name': None, + } + modified_since = 1234567890 + lp_bug_generator.return_value = iter([BUG]) + lp_module_exists.side_effect = iter([True]) + + expected = [] + + actual = list(bps.log(repo, modified_since)) + + lp_module_exists.assert_not_called() + self.assertEqual(expected, actual) + + @mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists') + @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') + def test_log_module_launchpad_name_with_alias(self, lp_bug_generator, + lp_module_exists): + repo = { + 'module': 'savanna', + 'launchpad_name': 'sahara', + 'aliases': ['someothername'], + } + modified_since = 1234567890 + list(bps.log(repo, modified_since)) + + assert lp_module_exists.asser_has_calls( + [mock.call('sahara'), mock.call('someothername')], any_order=True) + + @mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists') + @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') + def test_log_module_launchpad_name(self, lp_bug_generator, + lp_module_exists): + repo = { + 'module': 'savanna', + 'launchpad_name': 'sahara', + } + modified_since = 1234567890 + lp_bug_generator.return_value = iter([BUG]) + lp_module_exists.side_effect = iter([True]) expected = [{ 'assignee': 'slukjanov',