openstack/swift
Code Issues Proposed changes

Bump up sleep when expecting a timeout

Browse Source 
It seemed like some of the tests ment to exercise the proxy's timeout could
would occastionally race and the backend mock would be able to cough up a
chunk before the timeout fired in the proxy - typically resulting in a failed
assertion like " lalala" != 'lalala'.

We paramaterize the timeout value, and bump it up for those cases where we are
expecting to hit the timeout.  Timing shows this change is just as fast for
passing tests.  And if you want to break the node_timeout in
proxy.controller.base you can verify the tests are still just as effective -
if a bit slower to detect the failure path.

Fixes bug #1272509

Change-Id: Iaf91d9d551e94fc317a08e8c0ee02daeed331b60
This commit is contained in:
Clay Gerrard 2014-03-24 16:10:18 -07:00 committed by John Dickinson
parent 6145c57f4a
commit a3c4713cda
2 changed files with 38 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
test/unit/__init__.py
View File

@ -33,6 +33,7 @@ from hashlib import md5
from eventlet import sleep, Timeout
import logging.handlers
from httplib import HTTPException
from numbers import Number
class FakeRing(object):
@ -450,8 +451,11 @@ def fake_http_connect(*code_iter, **kwargs):
self.body = body
self.headers = headers or {}
self.timestamp = timestamp
if kwargs.get('slow') and isinstance(kwargs['slow'], list):
kwargs['slow'][0] -= 1
if 'slow' in kwargs and isinstance(kwargs['slow'], list):
try:
self._next_sleep = kwargs['slow'].pop(0)
except IndexError:
self._next_sleep = None
def getresponse(self):
if kwargs.get('raise_exc'):
@ -495,31 +499,39 @@ def fake_http_connect(*code_iter, **kwargs):
headers['x-container-timestamp'] = '1'
except StopIteration:
pass
if self.am_slow():
am_slow, value = self.get_slow()
if am_slow:
headers['content-length'] = '4'
headers.update(self.headers)
return headers.items()
def am_slow(self):
if kwargs.get('slow') and isinstance(kwargs['slow'], list):
return kwargs['slow'][0] >= 0
return bool(kwargs.get('slow'))
def get_slow(self):
if 'slow' in kwargs and isinstance(kwargs['slow'], list):
if self._next_sleep is not None:
return True, self._next_sleep
else:
return False, 0.01
if kwargs.get('slow') and isinstance(kwargs['slow'], Number):
return True, kwargs['slow']
return bool(kwargs.get('slow')), 0.1
def read(self, amt=None):
if self.am_slow():
am_slow, value = self.get_slow()
if am_slow:
if self.sent < 4:
self.sent += 1
sleep(0.1)
sleep(value)
return ' '
rv = self.body[:amt]
self.body = self.body[amt:]
return rv
def send(self, amt=None):
if self.am_slow():
am_slow, value = self.get_slow()
if am_slow:
if self.received < 4:
self.received += 1
sleep(0.1)
sleep(value)
def getheader(self, name, default=None):
return dict(self.getheaders()).get(name.lower(), default)

29
test/unit/proxy/test_server.py
View File

@ -1678,7 +1678,7 @@ class TestObjectController(unittest.TestCase):
dev['port'] = 1
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.app.update_request(req)
set_http_connect(200, 200, 200, slow=True)
set_http_connect(200, 200, 200, slow=0.1)
req.sent_size = 0
resp = req.get_response(self.app)
got_exc = False
@ -1688,7 +1688,7 @@ class TestObjectController(unittest.TestCase):
got_exc = True
self.assert_(not got_exc)
self.app.recoverable_node_timeout = 0.1
set_http_connect(200, 200, 200, slow=True)
set_http_connect(200, 200, 200, slow=1.0)
resp = req.get_response(self.app)
got_exc = False
try:
@ -1703,16 +1703,17 @@ class TestObjectController(unittest.TestCase):
self.app.update_request(req)
self.app.recoverable_node_timeout = 0.1
set_http_connect(200, 200, 200, slow=[3])
set_http_connect(200, 200, 200, slow=[1.0, 1.0, 1.0])
resp = req.get_response(self.app)
got_exc = False
try:
resp.body
self.assertEquals('', resp.body)
except ChunkReadTimeout:
got_exc = True
self.assert_(got_exc)
set_http_connect(200, 200, 200, body='lalala', slow=[2])
set_http_connect(200, 200, 200, body='lalala',
slow=[1.0, 1.0])
resp = req.get_response(self.app)
got_exc = False
try:
@ -1721,8 +1722,8 @@ class TestObjectController(unittest.TestCase):
got_exc = True
self.assert_(not got_exc)
set_http_connect(200, 200, 200, body='lalala', slow=[2],
etags=['a', 'a', 'a'])
set_http_connect(200, 200, 200, body='lalala',
slow=[1.0, 1.0], etags=['a', 'a', 'a'])
resp = req.get_response(self.app)
got_exc = False
try:
@ -1731,8 +1732,8 @@ class TestObjectController(unittest.TestCase):
got_exc = True
self.assert_(not got_exc)
set_http_connect(200, 200, 200, body='lalala', slow=[2],
etags=['a', 'b', 'a'])
set_http_connect(200, 200, 200, body='lalala',
slow=[1.0, 1.0], etags=['a', 'b', 'a'])
resp = req.get_response(self.app)
got_exc = False
try:
@ -1742,8 +1743,8 @@ class TestObjectController(unittest.TestCase):
self.assert_(not got_exc)
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
set_http_connect(200, 200, 200, body='lalala', slow=[2],
etags=['a', 'b', 'b'])
set_http_connect(200, 200, 200, body='lalala',
slow=[1.0, 1.0], etags=['a', 'b', 'b'])
resp = req.get_response(self.app)
got_exc = False
try:
@ -1772,7 +1773,7 @@ class TestObjectController(unittest.TestCase):
'Content-Type': 'text/plain'},
body=' ')
self.app.update_request(req)
set_http_connect(200, 200, 201, 201, 201, slow=True)
set_http_connect(200, 200, 201, 201, 201, slow=0.1)
resp = req.get_response(self.app)
self.assertEquals(resp.status_int, 201)
self.app.node_timeout = 0.1
@ -1782,7 +1783,7 @@ class TestObjectController(unittest.TestCase):
'Content-Type': 'text/plain'},
body=' ')
self.app.update_request(req)
set_http_connect(201, 201, 201, slow=True)
set_http_connect(201, 201, 201, slow=1.0)
resp = req.get_response(self.app)
self.assertEquals(resp.status_int, 503)
@ -5111,7 +5112,7 @@ class TestContainerController(unittest.TestCase):
with save_globals():
req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
self.app.node_timeout = 0.1
set_http_connect(200, 200, 200, body='abcdef', slow=[2])
set_http_connect(200, 200, 200, body='abcdef', slow=[1.0, 1.0])
resp = req.get_response(self.app)
got_exc = False
try: