zuul
/
nodepool
Code Issues Proposed changes

Use iterate_timeout in test waits 

This ensures that we don't wait forever for tests to complete tasks.
This is particularly useful if you've disabled the global test timeout.

Change-Id: I0141e62826c3594ed20605cac25e39091d1514e2
This commit is contained in:
Clark Boylan 2020-01-13 14:23:27 -08:00
parent 4ac8bf0cd9
commit 6276562939
2 changed files with 47 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
nodepool/nodeutils.py
View File

@ -29,13 +29,13 @@ log = logging.getLogger("nodepool.utils")
ITERATE_INTERVAL = 2 ITERATE_INTERVAL = 2
def iterate_timeout(max_seconds, exc, purpose): def iterate_timeout(max_seconds, exc, purpose, interval=ITERATE_INTERVAL):
start = time.time() start = time.time()
count = 0 count = 0
while (time.time() < start + max_seconds): while (time.time() < start + max_seconds):
count += 1 count += 1
yield count yield count
time.sleep(ITERATE_INTERVAL) time.sleep(interval)
raise exc("Timeout waiting for %s" % purpose) raise exc("Timeout waiting for %s" % purpose)

74
nodepool/tests/__init__.py
View File

@ -38,8 +38,11 @@ from nodepool import launcher
from nodepool import webapp from nodepool import webapp
from nodepool import zk from nodepool import zk
from nodepool.cmd.config_validator import ConfigValidator from nodepool.cmd.config_validator import ConfigValidator
from nodepool.nodeutils import iterate_timeout
TRUE_VALUES = ('true', '1', 'yes') TRUE_VALUES = ('true', '1', 'yes')
SECOND = 1
ONE_MINUTE = 60 * SECOND
class LoggingPopen(subprocess.Popen): class LoggingPopen(subprocess.Popen):
@ -224,7 +227,9 @@ class BaseTestCase(testtools.TestCase):
'pydevd.Writer', 'pydevd.Writer',
] ]
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Transient threads to finish",
interval=0.1):
done = True done = True
for t in threading.enumerate(): for t in threading.enumerate():
if t.name.startswith("Thread-"): if t.name.startswith("Thread-"):
@ -245,7 +250,6 @@ class BaseTestCase(testtools.TestCase):
done = False done = False
if done: if done:
return return
time.sleep(0.1)
def assertReportedStat(self, key, value=None, kind=None): def assertReportedStat(self, key, value=None, kind=None):
"""Check statsd output """Check statsd output
@ -268,8 +272,9 @@ class BaseTestCase(testtools.TestCase):
if value: if value:
self.assertNotEqual(kind, None) self.assertNotEqual(kind, None)
start = time.time() for _ in iterate_timeout(5 * SECOND, Exception,
while time.time() < (start + 5): "Find statsd event",
interval=0.1):
# Note our fake statsd just queues up results in a queue. # Note our fake statsd just queues up results in a queue.
# We just keep going through them until we find one that # We just keep going through them until we find one that
# matches, or fail out. If statsd pipelines are used, # matches, or fail out. If statsd pipelines are used,
@ -305,7 +310,6 @@ class BaseTestCase(testtools.TestCase):
# this key matches # this key matches
return True return True
time.sleep(0.1)
raise Exception("Key %s not found in reported stats" % key) raise Exception("Key %s not found in reported stats" % key)
@ -407,7 +411,9 @@ class DBTestCase(BaseTestCase):
time.sleep(0.1) time.sleep(0.1)
def waitForImage(self, provider_name, image_name, ignore_list=None): def waitForImage(self, provider_name, image_name, ignore_list=None):
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Image upload to be ready",
interval=1):
self.wait_for_threads() self.wait_for_threads()
image = self.zk.getMostRecentImageUpload(image_name, provider_name) image = self.zk.getMostRecentImageUpload(image_name, provider_name)
if image: if image:
@ -415,27 +421,28 @@ class DBTestCase(BaseTestCase):
break break
elif not ignore_list: elif not ignore_list:
break break
time.sleep(1)
self.wait_for_threads() self.wait_for_threads()
return image return image
def waitForUploadRecordDeletion(self, provider_name, image_name, def waitForUploadRecordDeletion(self, provider_name, image_name,
build_id, upload_id): build_id, upload_id):
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Image upload record deletion",
interval=1):
self.wait_for_threads() self.wait_for_threads()
uploads = self.zk.getUploads(image_name, build_id, provider_name) uploads = self.zk.getUploads(image_name, build_id, provider_name)
if not uploads or upload_id not in [u.id for u in uploads]: if not uploads or upload_id not in [u.id for u in uploads]:
break break
time.sleep(1)
self.wait_for_threads() self.wait_for_threads()
def waitForImageDeletion(self, provider_name, image_name, match=None): def waitForImageDeletion(self, provider_name, image_name, match=None):
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Image upload deletion",
interval=1):
self.wait_for_threads() self.wait_for_threads()
image = self.zk.getMostRecentImageUpload(image_name, provider_name) image = self.zk.getMostRecentImageUpload(image_name, provider_name)
if not image or (match and image != match): if not image or (match and image != match):
break break
time.sleep(1)
self.wait_for_threads() self.wait_for_threads()
def waitForBuild(self, image_name, build_id, states=None): def waitForBuild(self, image_name, build_id, states=None):
@ -444,12 +451,13 @@ class DBTestCase(BaseTestCase):
base = "-".join([image_name, build_id]) base = "-".join([image_name, build_id])
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Image build record to reach state",
interval=1):
self.wait_for_threads() self.wait_for_threads()
build = self.zk.getBuild(image_name, build_id) build = self.zk.getBuild(image_name, build_id)
if build and build.state in states: if build and build.state in states:
break break
time.sleep(1)
# We should only expect a dib manifest with a successful build. # We should only expect a dib manifest with a successful build.
while build.state == zk.READY: while build.state == zk.READY:
@ -465,34 +473,39 @@ class DBTestCase(BaseTestCase):
def waitForBuildDeletion(self, image_name, build_id): def waitForBuildDeletion(self, image_name, build_id):
base = "-".join([image_name, build_id]) base = "-".join([image_name, build_id])
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"DIB build files deletion",
interval=1):
self.wait_for_threads() self.wait_for_threads()
files = builder.DibImageFile.from_image_id( files = builder.DibImageFile.from_image_id(
self._config_images_dir.path, base) self._config_images_dir.path, base)
if not files: if not files:
break break
time.sleep(1)
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"DIB build file deletion leaks",
interval=1):
self.wait_for_threads() self.wait_for_threads()
# Now, check the disk to ensure we didn't leak any files. # Now, check the disk to ensure we didn't leak any files.
matches = glob.glob('%s/%s.*' % (self._config_images_dir.path, matches = glob.glob('%s/%s.*' % (self._config_images_dir.path,
base)) base))
if not matches: if not matches:
break break
time.sleep(1)
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Image build record deletion",
interval=1):
self.wait_for_threads() self.wait_for_threads()
build = self.zk.getBuild(image_name, build_id) build = self.zk.getBuild(image_name, build_id)
if not build: if not build:
break break
time.sleep(1)
self.wait_for_threads() self.wait_for_threads()
def waitForNodeDeletion(self, node): def waitForNodeDeletion(self, node):
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Node record deletion",
interval=1):
exists = False exists = False
for n in self.zk.nodeIterator(): for n in self.zk.nodeIterator():
if node.id == n.id: if node.id == n.id:
@ -500,17 +513,19 @@ class DBTestCase(BaseTestCase):
break break
if not exists: if not exists:
break break
time.sleep(1)
def waitForInstanceDeletion(self, manager, instance_id): def waitForInstanceDeletion(self, manager, instance_id):
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Cloud instance deletion",
interval=1):
servers = manager.listNodes() servers = manager.listNodes()
if not (instance_id in [s.id for s in servers]): if not (instance_id in [s.id for s in servers]):
break break
time.sleep(1)
def waitForNodeRequestLockDeletion(self, request_id): def waitForNodeRequestLockDeletion(self, request_id):
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Node request lock deletion",
interval=1):
exists = False exists = False
for lock_id in self.zk.getNodeRequestLockIDs(): for lock_id in self.zk.getNodeRequestLockIDs():
if request_id == lock_id: if request_id == lock_id:
@ -518,15 +533,15 @@ class DBTestCase(BaseTestCase):
break break
if not exists: if not exists:
break break
time.sleep(1)
def waitForNodes(self, label, count=1): def waitForNodes(self, label, count=1):
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Ready nodes",
interval=1):
self.wait_for_threads() self.wait_for_threads()
ready_nodes = self.zk.getReadyNodesOfTypes([label]) ready_nodes = self.zk.getReadyNodesOfTypes([label])
if label in ready_nodes and len(ready_nodes[label]) == count: if label in ready_nodes and len(ready_nodes[label]) == count:
break break
time.sleep(1)
self.wait_for_threads() self.wait_for_threads()
return ready_nodes[label] return ready_nodes[label]
@ -536,11 +551,12 @@ class DBTestCase(BaseTestCase):
''' '''
if states is None: if states is None:
states = (zk.FULFILLED, zk.FAILED) states = (zk.FULFILLED, zk.FAILED)
while True: for _ in iterate_timeout(ONE_MINUTE, Exception,
"Node request state transition",
interval=1):
req = self.zk.getNodeRequest(req.id) req = self.zk.getNodeRequest(req.id)
if req.state in states: if req.state in states:
break break
time.sleep(1)
return req return req