x/stackalytics
Code Issues Proposed changes

Retry on Gerrit errors

Browse Source 
In case of Gerrit communication errors do some number of retries.

Initially Stackalytics tried to retry always. But at high error rate
that resulted in connection leakage on Gerrit side. And as side-effect
Gerrit banned Stackalytics user.

Without any retries Stackalytics is not able to pull all data at a single
pass. So with this patch we allow to fail some constant number of times.

Change-Id: I8b92be57f82ecf3f847e0955bc618353c9d6b39a
Note: this partially reverts commit dced0aea28.
This commit is contained in:
Ilya Shakhat
2016-07-04 13:46:37 +03:00
parent 514240c990
commit 890b2774bf
2 changed files with 78 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
stackalytics/tests/unit/test_rcs.py
View File

@@ -124,3 +124,66 @@ class TestRcs(testtools.TestCase):
'--format JSON project:\'openstack/nova\' branch:master '
'limit:100 age:0s status:merged --comments'),
])
@mock.patch('paramiko.SSHClient')
@mock.patch('time.time')
def test_log_error_tolerated(self, mock_time, mock_client_cons):
mock_client = mock.Mock()
mock_client_cons.return_value = mock_client
mock_exec = mock.Mock()
mock_client.exec_command = mock_exec
mock_exec.side_effect = [
Exception,
('', [REVIEW_ONE, REVIEW_END_LINE], ''), # one review and summary
Exception,
('', [REVIEW_END_LINE], ''), # only summary = no more reviews
]
gerrit = rcs.Gerrit('uri')
repo = dict(organization='openstack', module='nova')
branch = 'master'
last_retrieval_time = 1444000000
mock_time.return_value = 1444333333
records = list(gerrit.log(repo, branch, last_retrieval_time))
self.assertEqual(1, len(records))
self.assertEqual('229382', records[0]['number'])
mock_client.exec_command.assert_has_calls([
mock.call('gerrit query --all-approvals --patch-sets '
'--format JSON project:\'openstack/nova\' branch:master '
'limit:100 age:0s'),
mock.call('gerrit query --all-approvals --patch-sets '
'--format JSON project:\'openstack/nova\' branch:master '
'limit:100 age:111111s'),
])
@mock.patch('paramiko.SSHClient')
@mock.patch('time.time')
def test_log_error_fatal(self, mock_time, mock_client_cons):
mock_client = mock.Mock()
mock_client_cons.return_value = mock_client
mock_exec = mock.Mock()
mock_client.exec_command = mock_exec
mock_exec.side_effect = [Exception] * rcs.SSH_ERRORS_LIMIT
gerrit = rcs.Gerrit('uri')
repo = dict(organization='openstack', module='nova')
branch = 'master'
last_retrieval_time = 1444000000
mock_time.return_value = 1444333333
try:
list(gerrit.log(repo, branch, last_retrieval_time))
self.fail('Gerrit.log should raise RcsException, but it did not')
except rcs.RcsException:
pass
mock_client.exec_command.assert_has_calls([
mock.call('gerrit query --all-approvals --patch-sets '
'--format JSON project:\'openstack/nova\' branch:master '
'limit:100 age:0s')] * rcs.SSH_ERRORS_LIMIT)