x/stackalytics
Code Issues Proposed changes

Merge "Allow to exclude module name from launchpad query"

Browse Source
This commit is contained in:
Zuul
2019-03-24 16:51:35 +00:00
committed by Gerrit Code Review
parent 7be485d230 8229f18f83
commit fbd22657db
6 changed files with 101 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
stackalytics/processor/bps.py
View File

@@ -70,7 +70,9 @@ def _log_module(module, primary_module, modified_since):
def log(repo, modified_since):
repo_module = repo['module']
modules = [repo_module] + repo.get('aliases', [])
modules = repo.get('aliases', [])
if repo['launchpad_name']:
modules.append(repo['launchpad_name'])
for module in modules:
if not launchpad_utils.lp_module_exists(module):

11
stackalytics/processor/default_data_processor.py
View File

@@ -85,9 +85,11 @@ def _retrieve_project_list_from_gerrit(project_source):
raise
git_base_uri = project_source.get('git_base_uri') or CONF.git_base_uri
use_launchpad_metrics = project_source.get('launchpad_metrics', True)
for repo in git_repos:
name = repo.split('/')[-1]
launchpad_name = name if use_launchpad_metrics else None
repo_uri = '%(git_base_uri)s/%(repo)s.git' % dict(
git_base_uri=git_base_uri, repo=repo)
yield {
@@ -100,6 +102,7 @@ def _retrieve_project_list_from_gerrit(project_source):
'gerrit_uri': uri,
'ssh_username': username,
'key_filename': key_filename,
'launchpad_name': launchpad_name,
}
@@ -117,14 +120,18 @@ def _retrieve_project_list_from_github(project_source):
LOG.error('Failed to retrieve list of projects from GitHub: %s',
e, exc_info=True)
raise
use_launchpad_metrics = project_source.get('launchpad_metrics', True)
for repo in github_repos:
name = repo.name.lower()
launchpad_name = name if use_launchpad_metrics else None
yield {
'branches': [project_source.get('default_branch', 'master')],
'module': repo.name.lower(),
'module': name,
'organization': organization,
'uri': repo.git_url,
'releases': []
'releases': [],
'launchpad_name': launchpad_name,
}

4
stackalytics/processor/lp.py
View File

@@ -47,7 +47,9 @@ def _log_module(module, primary_module):
def log(repo):
repo_module = repo['module']
modules = [repo_module] + repo.get('aliases', [])
modules = repo.get('aliases', [])
if repo['launchpad_name']:
modules.append(repo['launchpad_name'])
for module in modules:
if not launchpad_utils.lp_module_exists(module):

9
stackalytics/processor/main.py
View File

@@ -182,9 +182,12 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst):
_process_repo_vcs(repo, runtime_storage_inst, record_processor_inst)
_process_repo_bugs(repo, runtime_storage_inst, record_processor_inst)
_process_repo_blueprints(repo, runtime_storage_inst, record_processor_inst)
if repo['launchpad_name'] or repo.get('aliases'):
_process_repo_bugs(repo, runtime_storage_inst, record_processor_inst)
_process_repo_blueprints(repo, runtime_storage_inst,
record_processor_inst)
else:
LOG.info('Skipping launchpad data for repo: %s', repo['uri'])
if 'gerrit_uri' in repo:
_process_repo_reviews(repo, runtime_storage_inst,

6
stackalytics/processor/schema.py
View File

@@ -104,6 +104,9 @@ default_data = {
"module": {
"type": "string"
},
"launchpad_name": {
"type": ["string", "null"]
},
"releases": {
"type": "array",
"items": {
@@ -197,6 +200,9 @@ default_data = {
"module_group_id": {
"type": "string"
},
"launchpad_metrics": {
"type": "boolean"
},
"pattern": {
"type": "string"
}

79
stackalytics/tests/unit/test_bps.py
View File

@@ -165,7 +165,8 @@ class TestBps(testtools.TestCase):
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log(self, lp_bug_generator):
repo = {
'module': 'sahara'
'module': 'sahara',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG])
@@ -191,7 +192,8 @@ class TestBps(testtools.TestCase):
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log_released_not_committed(self, lp_bug_generator):
repo = {
'module': 'sahara'
'module': 'sahara',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([RELEASED_NOT_COMMITTED_BUG])
@@ -217,7 +219,8 @@ class TestBps(testtools.TestCase):
def test_log_additional_module(self, lp_bug_generator):
# bug linked to another project should not appear
repo = {
'module': 'sahara'
'module': 'sahara',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG, LINKED_BUG])
@@ -244,7 +247,8 @@ class TestBps(testtools.TestCase):
def test_log_additional_milestone(self, lp_bug_generator):
# bug linked to different milestone should be mapped to the release
repo = {
'module': 'sahara'
'module': 'sahara',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG, ANOTHER_MILESTONE_BUG])
@@ -288,11 +292,76 @@ class TestBps(testtools.TestCase):
# bug linked to another project should not appear
repo = {
'module': 'savanna',
'launchpad_name': None,
'aliases': ['sahara']
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG])
lp_module_exists.side_effect = iter([False, True])
lp_module_exists.side_effect = iter([True])
expected = [{
'assignee': 'slukjanov',
'date_created': 1433252154,
'date_fix_committed': 1433266265,
'date_fix_released': 1433266265,
'id': 'savanna/1458945',
'importance': 'Medium',
'module': 'savanna', # should be the same as primary module name
'owner': 'samueldmq',
'status': 'Fix Released',
'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"',
'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945'
}]
actual = list(bps.log(repo, modified_since))
self.assertEqual(expected, actual)
@mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists')
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log_module_launchpad_name_none(self, lp_bug_generator,
lp_module_exists):
repo = {
'module': 'savanna',
'launchpad_name': None,
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG])
lp_module_exists.side_effect = iter([True])
expected = []
actual = list(bps.log(repo, modified_since))
lp_module_exists.assert_not_called()
self.assertEqual(expected, actual)
@mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists')
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log_module_launchpad_name_with_alias(self, lp_bug_generator,
lp_module_exists):
repo = {
'module': 'savanna',
'launchpad_name': 'sahara',
'aliases': ['someothername'],
}
modified_since = 1234567890
list(bps.log(repo, modified_since))
assert lp_module_exists.asser_has_calls(
[mock.call('sahara'), mock.call('someothername')], any_order=True)
@mock.patch('stackalytics.processor.launchpad_utils.lp_module_exists')
@mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
def test_log_module_launchpad_name(self, lp_bug_generator,
lp_module_exists):
repo = {
'module': 'savanna',
'launchpad_name': 'sahara',
}
modified_since = 1234567890
lp_bug_generator.return_value = iter([BUG])
lp_module_exists.side_effect = iter([True])
expected = [{
'assignee': 'slukjanov',