Merge "Big Switch: Retry on 503 errors from backend"
This commit is contained in:
commit
3a4a1110b1
|
@ -35,6 +35,7 @@ import httplib
|
|||
import os
|
||||
import socket
|
||||
import ssl
|
||||
import time
|
||||
import weakref
|
||||
|
||||
import eventlet
|
||||
|
@ -73,6 +74,8 @@ ORCHESTRATION_SERVICE_ID = 'Neutron v2.0'
|
|||
HASH_MATCH_HEADER = 'X-BSN-BVS-HASH-MATCH'
|
||||
# error messages
|
||||
NXNETWORK = 'NXVNS'
|
||||
HTTP_SERVICE_UNAVAILABLE_RETRY_COUNT = 3
|
||||
HTTP_SERVICE_UNAVAILABLE_RETRY_INTERVAL = 3
|
||||
|
||||
|
||||
class RemoteRestError(exceptions.NeutronException):
|
||||
|
@ -428,10 +431,15 @@ class ServerPool(object):
|
|||
good_first = sorted(self.servers, key=lambda x: x.failed)
|
||||
first_response = None
|
||||
for active_server in good_first:
|
||||
ret = active_server.rest_call(action, resource, data, headers,
|
||||
timeout,
|
||||
reconnect=self.always_reconnect,
|
||||
hash_handler=hash_handler)
|
||||
for x in range(HTTP_SERVICE_UNAVAILABLE_RETRY_COUNT + 1):
|
||||
ret = active_server.rest_call(action, resource, data, headers,
|
||||
timeout,
|
||||
reconnect=self.always_reconnect,
|
||||
hash_handler=hash_handler)
|
||||
if ret[0] != httplib.SERVICE_UNAVAILABLE:
|
||||
break
|
||||
time.sleep(HTTP_SERVICE_UNAVAILABLE_RETRY_INTERVAL)
|
||||
|
||||
# If inconsistent, do a full synchronization
|
||||
if ret[0] == httplib.CONFLICT:
|
||||
if not self.get_topo_function:
|
||||
|
|
|
@ -374,6 +374,26 @@ class ServerManagerTests(test_rp.BigSwitchProxyPluginV2TestCase):
|
|||
self.assertFalse(pl.servers.server_failure((404,),
|
||||
ignore_codes=[404]))
|
||||
|
||||
def test_retry_on_unavailable(self):
|
||||
pl = manager.NeutronManager.get_plugin()
|
||||
with contextlib.nested(
|
||||
mock.patch(SERVERMANAGER + '.ServerProxy.rest_call',
|
||||
return_value=(httplib.SERVICE_UNAVAILABLE, 0, 0, 0)),
|
||||
mock.patch(SERVERMANAGER + '.time.sleep')
|
||||
) as (srestmock, tmock):
|
||||
# making a call should trigger retries with sleeps in between
|
||||
pl.servers.rest_call('GET', '/', '', None, [])
|
||||
rest_call = [mock.call('GET', '/', '', None, False, reconnect=True,
|
||||
hash_handler=mock.ANY)]
|
||||
rest_call_count = (
|
||||
servermanager.HTTP_SERVICE_UNAVAILABLE_RETRY_COUNT + 1)
|
||||
srestmock.assert_has_calls(rest_call * rest_call_count)
|
||||
sleep_call = [mock.call(
|
||||
servermanager.HTTP_SERVICE_UNAVAILABLE_RETRY_INTERVAL)]
|
||||
# should sleep 1 less time than the number of calls
|
||||
sleep_call_count = rest_call_count - 1
|
||||
tmock.assert_has_calls(sleep_call * sleep_call_count)
|
||||
|
||||
def test_conflict_triggers_sync(self):
|
||||
pl = manager.NeutronManager.get_plugin()
|
||||
with mock.patch(
|
||||
|
|
Loading…
Reference in New Issue