From 8229f18f83ca252111c1281b2a9018ade7c4c140 Mon Sep 17 00:00:00 2001 From: Andrii Ostapenko Date: Thu, 7 Mar 2019 14:01:33 -0600 Subject: [PATCH] Allow to exclude module name from launchpad query Current implementation does not allow to exclude module name from list of repo launchpad names, that does not cover case when launchpad module with module name exists, but has no relation to repository. This commit adds ability to specify if launchpad should be queried for projects in project source, as well as to specify launchpad name for staticly defined repos without changing default behavior. Change-Id: I7bdc028ee6f962e9e903ede7917180b8c8c12446 --- stackalytics/processor/bps.py | 4 +- .../processor/default_data_processor.py | 11 ++- stackalytics/processor/lp.py | 4 +- stackalytics/processor/main.py | 9 ++- stackalytics/processor/schema.py | 6 ++ stackalytics/tests/unit/test_bps.py | 79 +++++++++++++++++-- 6 files changed, 101 insertions(+), 12 deletions(-) diff --git a/stackalytics/processor/bps.py b/stackalytics/processor/bps.py index f6c07adcf..4c93ff798 100644 --- a/stackalytics/processor/bps.py +++ b/stackalytics/processor/bps.py @@ -70,7 +70,9 @@ def _log_module(module, primary_module, modified_since): def log(repo, modified_since): repo_module = repo['module'] - modules = [repo_module] + repo.get('aliases', []) + modules = repo.get('aliases', []) + if repo['launchpad_name']: + modules.append(repo['launchpad_name']) for module in modules: if not launchpad_utils.lp_module_exists(module): diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py index 5cf59f20b..02521ffbe 100644 --- a/stackalytics/processor/default_data_processor.py +++ b/stackalytics/processor/default_data_processor.py @@ -85,9 +85,11 @@ def _retrieve_project_list_from_gerrit(project_source): raise git_base_uri = project_source.get('git_base_uri') or CONF.git_base_uri + use_launchpad_metrics = project_source.get('launchpad_metrics', True) for repo in git_repos: name = repo.split('/')[-1] + launchpad_name = name if use_launchpad_metrics else None repo_uri = '%(git_base_uri)s/%(repo)s.git' % dict( git_base_uri=git_base_uri, repo=repo) yield { @@ -100,6 +102,7 @@ def _retrieve_project_list_from_gerrit(project_source): 'gerrit_uri': uri, 'ssh_username': username, 'key_filename': key_filename, + 'launchpad_name': launchpad_name, } @@ -115,14 +118,18 @@ def _retrieve_project_list_from_github(project_source): LOG.error('Failed to retrieve list of projects from GitHub: %s', e, exc_info=True) raise + use_launchpad_metrics = project_source.get('launchpad_metrics', True) for repo in github_repos: + name = repo.name.lower() + launchpad_name = name if use_launchpad_metrics else None yield { 'branches': [project_source.get('default_branch', 'master')], - 'module': repo.name.lower(), + 'module': name, 'organization': organization, 'uri': repo.git_url, - 'releases': [] + 'releases': [], + 'launchpad_name': launchpad_name, } diff --git a/stackalytics/processor/lp.py b/stackalytics/processor/lp.py index f35f6f52e..186edbb2d 100644 --- a/stackalytics/processor/lp.py +++ b/stackalytics/processor/lp.py @@ -47,7 +47,9 @@ def _log_module(module, primary_module): def log(repo): repo_module = repo['module'] - modules = [repo_module] + repo.get('aliases', []) + modules = repo.get('aliases', []) + if repo['launchpad_name']: + modules.append(repo['launchpad_name']) for module in modules: if not launchpad_utils.lp_module_exists(module): diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 498817eff..b120e0ebe 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -182,9 +182,12 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst): _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst) - _process_repo_bugs(repo, runtime_storage_inst, record_processor_inst) - - _process_repo_blueprints(repo, runtime_storage_inst, record_processor_inst) + if repo['launchpad_name'] or repo.get('aliases'): + _process_repo_bugs(repo, runtime_storage_inst, record_processor_inst) + _process_repo_blueprints(repo, runtime_storage_inst, + record_processor_inst) + else: + LOG.info('Skipping launchpad data for repo: %s', repo['uri']) if 'gerrit_uri' in repo: _process_repo_reviews(repo, runtime_storage_inst, diff --git a/stackalytics/processor/schema.py b/stackalytics/processor/schema.py index 65b220c66..3c79f8611 100644 --- a/stackalytics/processor/schema.py +++ b/stackalytics/processor/schema.py @@ -101,6 +101,9 @@ default_data = { "module": { "type": "string" }, + "launchpad_name": { + "type": ["string", "null"] + }, "releases": { "type": "array", "items": { @@ -194,6 +197,9 @@ default_data = { "module_group_id": { "type": "string" }, + "launchpad_metrics": { + "type": "boolean" + }, "pattern": { "type": "string" } diff --git a/stackalytics/tests/unit/test_bps.py b/stackalytics/tests/unit/test_bps.py index df58586a2..f581f59b3 100644 --- a/stackalytics/tests/unit/test_bps.py +++ b/stackalytics/tests/unit/test_bps.py @@ -165,7 +165,8 @@ class TestBps(testtools.TestCase): @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') def test_log(self, lp_bug_generator): repo = { - 'module': 'sahara' + 'module': 'sahara', + 'launchpad_name': 'sahara', } modified_since = 1234567890 lp_bug_generator.return_value = iter([BUG]) @@ -191,7 +192,8 @@ class TestBps(testtools.TestCase): @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') def test_log_released_not_committed(self, lp_bug_generator): repo = { - 'module': 'sahara' + 'module': 'sahara', + 'launchpad_name': 'sahara', } modified_since = 1234567890 lp_bug_generator.return_value = iter([RELEASED_NOT_COMMITTED_BUG]) @@ -217,7 +219,8 @@ class TestBps(testtools.TestCase): def test_log_additional_module(self, lp_bug_generator): # bug linked to another project should not appear repo = { - 'module': 'sahara' + 'module': 'sahara', + 'launchpad_name': 'sahara', } modified_since = 1234567890 lp_bug_generator.return_value = iter([BUG, LINKED_BUG]) @@ -244,7 +247,8 @@ class TestBps(testtools.TestCase): def test_log_additional_milestone(self, lp_bug_generator): # bug linked to different milestone should be mapped to the release repo = { - 'module': 'sahara' + 'module': 'sahara', + 'launchpad_name': 'sahara', } modified_since = 1234567890 lp_bug_generator.return_value = iter([BUG, ANOTHER_MILESTONE_BUG]) @@ -288,11 +292,76 @@ class TestBps(testtools.TestCase): # bug linked to another project should not appear repo = { 'module': 'savanna', + 'launchpad_name': None, 'aliases': ['sahara'] } modified_since = 1234567890 lp_bug_generator.return_value = iter([BUG]) - lp_module_exists.side_effect = iter([False, True]) + lp_module_exists.side_effect = iter([True]) + + expected = [{ + 'assignee': 'slukjanov', + 'date_created': 1433252154, + 'date_fix_committed': 1433266265, + 'date_fix_released': 1433266265, + 'id': 'savanna/1458945', + 'importance': 'Medium', + 'module': 'savanna', # should be the same as primary module name + 'owner': 'samueldmq', + 'status': 'Fix Released', + 'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"', + 'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945' + }] + + actual = list(bps.log(repo, modified_since)) + + self.assertEqual(expected, actual) + + @mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists') + @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') + def test_log_module_launchpad_name_none(self, lp_bug_generator, + lp_module_exists): + repo = { + 'module': 'savanna', + 'launchpad_name': None, + } + modified_since = 1234567890 + lp_bug_generator.return_value = iter([BUG]) + lp_module_exists.side_effect = iter([True]) + + expected = [] + + actual = list(bps.log(repo, modified_since)) + + lp_module_exists.assert_not_called() + self.assertEqual(expected, actual) + + @mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists') + @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') + def test_log_module_launchpad_name_with_alias(self, lp_bug_generator, + lp_module_exists): + repo = { + 'module': 'savanna', + 'launchpad_name': 'sahara', + 'aliases': ['someothername'], + } + modified_since = 1234567890 + list(bps.log(repo, modified_since)) + + assert lp_module_exists.asser_has_calls( + [mock.call('sahara'), mock.call('someothername')], any_order=True) + + @mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists') + @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator') + def test_log_module_launchpad_name(self, lp_bug_generator, + lp_module_exists): + repo = { + 'module': 'savanna', + 'launchpad_name': 'sahara', + } + modified_since = 1234567890 + lp_bug_generator.return_value = iter([BUG]) + lp_module_exists.side_effect = iter([True]) expected = [{ 'assignee': 'slukjanov',