diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py index a81435bd5..012d1302b 100644 --- a/stackalytics/processor/rcs.py +++ b/stackalytics/processor/rcs.py @@ -26,6 +26,7 @@ DEFAULT_PORT = 29418 GERRIT_URI_PREFIX = r'^gerrit:\/\/' PAGE_LIMIT = 100 REQUEST_COUNT_LIMIT = 20 +SSH_ERRORS_LIMIT = 5 class RcsException(Exception): @@ -70,6 +71,7 @@ class Gerrit(Rcs): self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.request_count = 0 + self.error_count = 0 def setup(self, **kwargs): self.key_filename = kwargs.get('key_filename') @@ -117,7 +119,17 @@ class Gerrit(Rcs): except Exception as e: LOG.error('Error %(error)s while execute command %(cmd)s', {'error': e, 'cmd': cmd}, exc_info=True) - raise RcsException('Failed to execute command: %s', cmd) + self.request_count = REQUEST_COUNT_LIMIT + raise RcsException(e) + + def _exec_command_with_retrial(self, cmd): + while self.error_count < SSH_ERRORS_LIMIT: + try: + return self._exec_command(cmd) + except RcsException: + self.error_count += 1 + + raise RcsException('Too many SSH errors, aborting') def _poll_reviews(self, project_organization, module, branch, last_retrieval_time, status=None, grab_comments=False): @@ -136,7 +148,7 @@ class Gerrit(Rcs): age=age, status=status, grab_comments=grab_comments) LOG.debug('Executing command: %s', cmd) - exec_result = self._exec_command(cmd) + exec_result = self._exec_command_with_retrial(cmd) if not exec_result: break stdin, stdout, stderr = exec_result @@ -162,7 +174,7 @@ class Gerrit(Rcs): yield review def get_project_list(self): - exec_result = self._exec_command('gerrit ls-projects') + exec_result = self._exec_command_with_retrial('gerrit ls-projects') if not exec_result: raise RcsException("Gerrit returned no projects") stdin, stdout, stderr = exec_result diff --git a/stackalytics/tests/unit/test_rcs.py b/stackalytics/tests/unit/test_rcs.py index f2f343efd..7a77f79d9 100644 --- a/stackalytics/tests/unit/test_rcs.py +++ b/stackalytics/tests/unit/test_rcs.py @@ -124,3 +124,66 @@ class TestRcs(testtools.TestCase): '--format JSON project:\'openstack/nova\' branch:master ' 'limit:100 age:0s status:merged --comments'), ]) + + @mock.patch('paramiko.SSHClient') + @mock.patch('time.time') + def test_log_error_tolerated(self, mock_time, mock_client_cons): + mock_client = mock.Mock() + mock_client_cons.return_value = mock_client + + mock_exec = mock.Mock() + mock_client.exec_command = mock_exec + mock_exec.side_effect = [ + Exception, + ('', [REVIEW_ONE, REVIEW_END_LINE], ''), # one review and summary + Exception, + ('', [REVIEW_END_LINE], ''), # only summary = no more reviews + ] + + gerrit = rcs.Gerrit('uri') + + repo = dict(organization='openstack', module='nova') + branch = 'master' + last_retrieval_time = 1444000000 + mock_time.return_value = 1444333333 + records = list(gerrit.log(repo, branch, last_retrieval_time)) + + self.assertEqual(1, len(records)) + self.assertEqual('229382', records[0]['number']) + + mock_client.exec_command.assert_has_calls([ + mock.call('gerrit query --all-approvals --patch-sets ' + '--format JSON project:\'openstack/nova\' branch:master ' + 'limit:100 age:0s'), + mock.call('gerrit query --all-approvals --patch-sets ' + '--format JSON project:\'openstack/nova\' branch:master ' + 'limit:100 age:111111s'), + ]) + + @mock.patch('paramiko.SSHClient') + @mock.patch('time.time') + def test_log_error_fatal(self, mock_time, mock_client_cons): + mock_client = mock.Mock() + mock_client_cons.return_value = mock_client + + mock_exec = mock.Mock() + mock_client.exec_command = mock_exec + mock_exec.side_effect = [Exception] * rcs.SSH_ERRORS_LIMIT + + gerrit = rcs.Gerrit('uri') + + repo = dict(organization='openstack', module='nova') + branch = 'master' + last_retrieval_time = 1444000000 + mock_time.return_value = 1444333333 + + try: + list(gerrit.log(repo, branch, last_retrieval_time)) + self.fail('Gerrit.log should raise RcsException, but it did not') + except rcs.RcsException: + pass + + mock_client.exec_command.assert_has_calls([ + mock.call('gerrit query --all-approvals --patch-sets ' + '--format JSON project:\'openstack/nova\' branch:master ' + 'limit:100 age:0s')] * rcs.SSH_ERRORS_LIMIT)