opendev/elastic-recheck
Code Issues Proposed changes

Retry change queries on a 502 response

Browse Source 
When gerrit is running slow we get 502 responses
back which kills the graph builder. We can retry
these requests from the client to keep going. Generally
a single retry fixes it.

Change-Id: I745d7c9b80ab8861972193d82c037df76af69e06
This commit is contained in:
Matt Riedemann
2017-10-09 18:45:56 -04:00
parent e6eb1073a2
commit 57aab15c11
2 changed files with 20 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
elastic_recheck/cmd/graph.py
View File

@@ -74,11 +74,16 @@ def get_launchpad_bug(bug):
return bugdata return bugdata
def get_open_reviews(bug_number): def get_open_reviews(bug_number, attempt=0):
"return list of open gerrit reviews for a given bug.""" """return list of open gerrit reviews for a given bug."""
r = requests.get("https://review.openstack.org:443/changes/" r = requests.get("https://review.openstack.org:443/changes/"
"?q=status:open++message:`%s`+NOT+" "?q=status:open++message:`%s`+NOT+"
"project:openstack-infra/elastic-recheck" % bug_number) "project:openstack-infra/elastic-recheck" % bug_number)
# If we got a proxy error let's retry until we're out of attempts.
if r.status_code == 502 and attempt < 3:
LOG.debug('Retry changes query for bug %s. Attempt %s of 3.',
bug_number, (attempt + 1))
return get_open_reviews(bug_number, attempt + 1)
# strip off first few chars because 'the JSON response body starts with a # strip off first few chars because 'the JSON response body starts with a
# magic prefix line that must be stripped before feeding the rest of the # magic prefix line that must be stripped before feeding the rest of the
# response body to a JSON parser' # response body to a JSON parser'

14
elastic_recheck/tests/unit/test_graph.py
View File

@@ -17,10 +17,11 @@ from elastic_recheck.tests import unit
class FakeResponse(object): class FakeResponse(object):
def __init__(self, response_text): def __init__(self, response_text, status_code=200):
super(FakeResponse, self).__init__() super(FakeResponse, self).__init__()
# magic gerrit prefix # magic gerrit prefix
self.text = ")]}'\n" + response_text self.text = ")]}'\n" + response_text
self.status_code = status_code
class TestGraphCmd(unit.UnitTestCase): class TestGraphCmd(unit.UnitTestCase):
@@ -28,6 +29,7 @@ class TestGraphCmd(unit.UnitTestCase):
with mock.patch('requests.get') as mock_get: with mock.patch('requests.get') as mock_get:
mock_get.return_value = FakeResponse("[]\n") mock_get.return_value = FakeResponse("[]\n")
self.assertEqual(graph.get_open_reviews('1353131'), []) self.assertEqual(graph.get_open_reviews('1353131'), [])
mock_get.assert_called_once()
def test_get_open_reviews(self): def test_get_open_reviews(self):
with mock.patch('requests.get') as mock_get: with mock.patch('requests.get') as mock_get:
@@ -35,3 +37,13 @@ class TestGraphCmd(unit.UnitTestCase):
'gerrit-bug-query.json') as f: 'gerrit-bug-query.json') as f:
mock_get.return_value = FakeResponse(f.read()) mock_get.return_value = FakeResponse(f.read())
self.assertEqual(graph.get_open_reviews('1288393'), [113009]) self.assertEqual(graph.get_open_reviews('1288393'), [113009])
def test_get_open_reviews_502_retry(self):
"""Tests that we retry if we get a 502 response from the server."""
with mock.patch('requests.get') as mock_get:
mock_get.side_effect = (
FakeResponse("bad proxy gateway", status_code=502),
FakeResponse("[]\n"))
self.assertEqual(graph.get_open_reviews('1353131'), [])
# We call twice because we retried once.
self.assertEqual(2, mock_get.call_count)