x/stackalytics
Code Issues Proposed changes

Allow to exclude module name from launchpad query

Browse Source 
Current implementation does not allow to exclude module name from list
of repo launchpad names, that does not cover case when launchpad module
with module name exists, but has no relation to repository.

This commit adds ability to specify if launchpad should be queried for
projects in project source, as well as to specify launchpad name for
staticly defined repos without changing default behavior.

Change-Id: I7bdc028ee6f962e9e903ede7917180b8c8c12446
This commit is contained in:
Andrii Ostapenko
2019-03-07 14:01:33 -06:00
parent 2b94ab032d
commit 8229f18f83
6 changed files with 101 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
stackalytics/processor/bps.py
View File

@@ -70,7 +70,9 @@ def _log_module(module, primary_module, modified_since):
def log(repo, modified_since):
repo_module = repo['module']
modules = [repo_module] + repo.get('aliases', [])
modules = repo.get('aliases', [])
if repo['launchpad_name']:
modules.append(repo['launchpad_name'])
for module in modules:
if not launchpad_utils.lp_module_exists(module):

11
stackalytics/processor/default_data_processor.py
View File

@@ -85,9 +85,11 @@ def _retrieve_project_list_from_gerrit(project_source):
raise
git_base_uri = project_source.get('git_base_uri') or CONF.git_base_uri
use_launchpad_metrics = project_source.get('launchpad_metrics', True)
for repo in git_repos:
name = repo.split('/')[-1]
launchpad_name = name if use_launchpad_metrics else None
repo_uri = '%(git_base_uri)s/%(repo)s.git' % dict(
git_base_uri=git_base_uri, repo=repo)
yield {
@@ -100,6 +102,7 @@ def _retrieve_project_list_from_gerrit(project_source):
'gerrit_uri': uri,
'ssh_username': username,
'key_filename': key_filename,
'launchpad_name': launchpad_name,
}
@@ -115,14 +118,18 @@ def _retrieve_project_list_from_github(project_source):
LOG.error('Failed to retrieve list of projects from GitHub: %s',
e, exc_info=True)
raise
use_launchpad_metrics = project_source.get('launchpad_metrics', True)
for repo in github_repos:
name = repo.name.lower()
launchpad_name = name if use_launchpad_metrics else None
yield {
'branches': [project_source.get('default_branch', 'master')],
'module': repo.name.lower(),
'module': name,
'organization': organization,
'uri': repo.git_url,
'releases': []
'releases': [],
'launchpad_name': launchpad_name,
}

4
stackalytics/processor/lp.py
View File

@@ -47,7 +47,9 @@ def _log_module(module, primary_module):
def log(repo):
repo_module = repo['module']
modules = [repo_module] + repo.get('aliases', [])
modules = repo.get('aliases', [])
if repo['launchpad_name']:
modules.append(repo['launchpad_name'])
for module in modules:
if not launchpad_utils.lp_module_exists(module):

9
stackalytics/processor/main.py
View File

@@ -182,9 +182,12 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst):
_process_repo_vcs(repo, runtime_storage_inst, record_processor_inst)
_process_repo_bugs(repo, runtime_storage_inst, record_processor_inst)
_process_repo_blueprints(repo, runtime_storage_inst, record_processor_inst)
if repo['launchpad_name'] or repo.get('aliases'):
_process_repo_bugs(repo, runtime_storage_inst, record_processor_inst)
_process_repo_blueprints(repo, runtime_storage_inst,
record_processor_inst)
else:
LOG.info('Skipping launchpad data for repo: %s', repo['uri'])
if 'gerrit_uri' in repo:
_process_repo_reviews(repo, runtime_storage_inst,

6
stackalytics/processor/schema.py
View File

@@ -101,6 +101,9 @@ default_data = {
"module": {
"type": "string"
},
"launchpad_name": {
"type": ["string", "null"]
},
"releases": {
"type": "array",
"items": {
@@ -194,6 +197,9 @@ default_data = {
"module_group_id": {
"type": "string"
},
"launchpad_metrics": {
"type": "boolean"
},
"pattern": {
"type": "string"
}

79
stackalytics/tests/unit/test_bps.py
View File

@@ -165,7 +165,8 @@ class TestBps(testtools.TestCase):
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log(self, lp_bug_generator):
repo = {
'module': 'sahara'
'module': 'sahara',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG])
@@ -191,7 +192,8 @@ class TestBps(testtools.TestCase):
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log_released_not_committed(self, lp_bug_generator):
repo = {
'module': 'sahara'
'module': 'sahara',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([RELEASED_NOT_COMMITTED_BUG])
@@ -217,7 +219,8 @@ class TestBps(testtools.TestCase):
def test_log_additional_module(self, lp_bug_generator):
# bug linked to another project should not appear
repo = {
'module': 'sahara'
'module': 'sahara',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG, LINKED_BUG])
@@ -244,7 +247,8 @@ class TestBps(testtools.TestCase):
def test_log_additional_milestone(self, lp_bug_generator):
# bug linked to different milestone should be mapped to the release
repo = {
'module': 'sahara'
'module': 'sahara',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG, ANOTHER_MILESTONE_BUG])
@@ -288,11 +292,76 @@ class TestBps(testtools.TestCase):
# bug linked to another project should not appear
repo = {
'module': 'savanna',
'launchpad_name': None,
'aliases': ['sahara']
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG])
lp_module_exists.side_effect = iter([False, True])
lp_module_exists.side_effect = iter([True])
expected = [{
'assignee': 'slukjanov',
'date_created': 1433252154,
'date_fix_committed': 1433266265,
'date_fix_released': 1433266265,
'id': 'savanna/1458945',
'importance': 'Medium',
'module': 'savanna', # should be the same as primary module name
'owner': 'samueldmq',
'status': 'Fix Released',
'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"',
'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945'
}]
actual = list(bps.log(repo, modified_since))
self.assertEqual(expected, actual)
@mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists')
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log_module_launchpad_name_none(self, lp_bug_generator,
lp_module_exists):
repo = {
'module': 'savanna',
'launchpad_name': None,
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG])
lp_module_exists.side_effect = iter([True])
expected = []
actual = list(bps.log(repo, modified_since))
lp_module_exists.assert_not_called()
self.assertEqual(expected, actual)
@mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists')
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log_module_launchpad_name_with_alias(self, lp_bug_generator,
lp_module_exists):
repo = {
'module': 'savanna',
'launchpad_name': 'sahara',
'aliases': ['someothername'],
}
modified_since = 1234567890
list(bps.log(repo, modified_since))
assert lp_module_exists.asser_has_calls(
[mock.call('sahara'), mock.call('someothername')], any_order=True)
@mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists')
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log_module_launchpad_name(self, lp_bug_generator,
lp_module_exists):
repo = {
'module': 'savanna',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG])
lp_module_exists.side_effect = iter([True])
expected = [{
'assignee': 'slukjanov',