diff --git a/stackalytics/dashboard/helpers.py b/stackalytics/dashboard/helpers.py index e7a1cc22e..a134f8edf 100644 --- a/stackalytics/dashboard/helpers.py +++ b/stackalytics/dashboard/helpers.py @@ -297,6 +297,9 @@ def make_blueprint_link(module, name): def make_commit_message(record): s = record['message'] module = record['module'] + # NOTE(aostapenko) Keeping default value here not to brake links + # with existing storage data + gerrit_hostname = record.get('gerrit_hostname', 'review.openstack.org') s = utils.format_text(s) @@ -307,8 +310,10 @@ def make_commit_message(record): s = re.sub(re.compile('(bug[\s#:]*)([\d]{5,7})', flags=re.IGNORECASE), r'\1\2', s) + # NOTE(aostapenko) Setting http here as it's common practice to redirect + # http -> https, but not vice versa s = re.sub(r'\s+(I[0-9a-f]{40})', - r' \1', s) s = utils.unwrap_text(s) diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index cb3e4609e..498817eff 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -71,12 +71,16 @@ def _merge_commits(original, new): return True -def _record_typer(record_iterator, record_type): +def _param_adder(record_iterator, key, value): for record in record_iterator: - record['record_type'] = record_type + record[key] = value yield record +def _record_typer(record_iterator, record_type): + return _param_adder(record_iterator, 'record_type', record_type) + + def _get_repo_branches(repo): return ({repo.get('default_branch', 'master')} | set(r['branch'] for r in repo.get('releases', []) @@ -147,6 +151,9 @@ def _process_repo_reviews(repo, runtime_storage_inst, record_processor_inst): def _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst): vcs_inst = vcs.get_vcs(repo, CONF.sources_root) vcs_inst.fetch() + gerrit_hostname, _ = rcs.get_socket_tuple_from_uri( + repo.get('gerrit_uri', CONF.review_uri) + ) for branch in _get_repo_branches(repo): LOG.info('Processing commits in repo: %s, branch: %s', @@ -157,7 +164,10 @@ def _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst): last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) - commit_iterator_typed = _record_typer(commit_iterator, 'commit') + commit_iterator_review = _param_adder( + commit_iterator, 'gerrit_hostname', gerrit_hostname + ) + commit_iterator_typed = _record_typer(commit_iterator_review, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records( diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py index 5503a20fc..f987a9591 100644 --- a/stackalytics/processor/rcs.py +++ b/stackalytics/processor/rcs.py @@ -29,6 +29,17 @@ REQUEST_COUNT_LIMIT = 20 SSH_ERRORS_LIMIT = 10 +def get_socket_tuple_from_uri(uri): + stripped = re.sub(GERRIT_URI_PREFIX, '', uri) + if stripped: + hostname, semicolon, port = stripped.partition(':') + if not port: + port = DEFAULT_PORT + else: + raise RcsException('Invalid rcs uri %s' % uri) + return hostname, port + + class RcsException(Exception): pass @@ -57,13 +68,7 @@ class Gerrit(Rcs): def __init__(self, uri): super(Gerrit, self).__init__() - stripped = re.sub(GERRIT_URI_PREFIX, '', uri) - if stripped: - self.hostname, semicolon, self.port = stripped.partition(':') - if not self.port: - self.port = DEFAULT_PORT - else: - raise RcsException('Invalid rcs uri %s' % uri) + self.hostname, self.port = get_socket_tuple_from_uri(uri) self.key_filename = None self.username = None diff --git a/stackalytics/tests/unit/test_web_utils.py b/stackalytics/tests/unit/test_web_utils.py index 9f60620f6..aed42442d 100644 --- a/stackalytics/tests/unit/test_web_utils.py +++ b/stackalytics/tests/unit/test_web_utils.py @@ -46,7 +46,42 @@ returned. Fixes bug \ 1076801 ''' + ( - 'Change-Id: ' + 'Ie49ccd2138905e178843b375a9b16c3fe572d1db') + + observed = helpers.make_commit_message(record) + + self.assertEqual(expected, observed, + 'Commit message should be processed correctly') + + def test_make_commit_message_gerrit_host(self): + message = ''' +During finish_migration the manager calls initialize_connection but doesn't +update the block_device_mapping with the potentially new connection_info +returned. + + +Fixes bug 1076801 +Change-Id: Ie49ccd2138905e178843b375a9b16c3fe572d1db''' + + module = 'test' + + gerrit = 'someothergerrit.org' + record = { + 'message': message, + 'module': module, + 'gerrit_hostname': gerrit, + } + + expected = '''\ +During finish_migration the manager calls initialize_connection but doesn't \ +update the block_device_mapping with the potentially new connection_info \ +returned. +Fixes bug \ +1076801 +''' + ( + 'Change-Id: ' 'Ie49ccd2138905e178843b375a9b16c3fe572d1db') @@ -73,7 +108,7 @@ Implemented new driver for Cinder <: Implements Blueprint ''' + ( 'super-driver' + '\n' + - 'Change-Id: ' 'Ie49ccd2138905e178843b375a9b16c3fe572d1db')