From e732fec5bf3f97800bf271e7f806b92b604b18b6 Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Thu, 10 Oct 2019 14:47:27 -0400 Subject: [PATCH] Fix builder shutdown race in tests The builder intentionally does not attempt to shutdown the uploader threads since that could take an unreasonable amount of time. This causes a race in our tests where we can shutdown the ZooKeeper connection while the upload thread is still in progress, which can cause the test to fail with a ZooKeeper error. This adds uploader thread cleanup for the builder used in tests. Change-Id: I25d4b52e17501e5dc6543adef585dd3b86bd70f9 --- nodepool/tests/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nodepool/tests/__init__.py b/nodepool/tests/__init__.py index a452af935..a2da35c9b 100644 --- a/nodepool/tests/__init__.py +++ b/nodepool/tests/__init__.py @@ -311,6 +311,8 @@ class BaseTestCase(testtools.TestCase): class BuilderFixture(fixtures.Fixture): + log = logging.getLogger("tests.BuilderFixture") + def __init__(self, configfile, cleanup_interval, securefile=None, num_uploaders=1): super(BuilderFixture, self).__init__() @@ -332,6 +334,13 @@ class BuilderFixture(fixtures.Fixture): self.addCleanup(self.cleanup) def cleanup(self): + # The NodePoolBuilder.stop() method does not intentionally stop the + # upload workers for reasons documented in that method. But we can + # safely do so in tests. + for worker in self.builder._upload_workers: + worker.shutdown() + worker.join() + self.log.debug("Stopped worker %s", worker.name) self.builder.stop()