From b0a40f0b472142ec64636b519507658d4e1f35e1 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 14 Mar 2023 15:28:39 -0700 Subject: [PATCH] Use image cache when launching nodes We consult ZooKeeper to determine the most recent image upload when we decide whether we should accept or decline a request. If we accept the request, we also consult it again for the same information when we start building the node. In both cases, we can use the cache to avoid what may potentially be (especially in the case of a large number of images or uploads) quite a lot of ZK requests. Our cache should be almost up to date (typically milliseconds, or at the worst, seconds behind), and the worst case is equivalent to what would happen if an image build took just a few seconds longer. The tradeoff is worth it. Similarly, when we create min-ready requests, we can also consult the cache. With those 3 changes, all references to getMostRecentImageUpload in Nodepool use the cache. The original un-cached method is kept as well, because there are an enormous number of references to it in the unit tests and they don't have caching enabled. In order to reduce the chances of races in many tests, the startup sequence is normalized to: 1) start the builder 2) wait for an image to be available 3) start the launcher 4) check that the image cache in the launcher matches what is actually in ZK This sequence (apart from #4) was already used by a minority of tests (mostly newer tests). Older tests have been updated. A helper method, startPool, implements #4 and additionally includes the wait_for_config method which was used by a random assortment of tests. Change-Id: Iac1ff8adfbdb8eb9a286929a59cf07cd0b4ac7ad --- nodepool/driver/statemachine.py | 4 +- nodepool/launcher.py | 3 +- nodepool/tests/__init__.py | 32 ++- nodepool/tests/unit/test_builder.py | 4 +- nodepool/tests/unit/test_commands.py | 54 ++-- nodepool/tests/unit/test_driver_aws.py | 22 +- nodepool/tests/unit/test_driver_azure.py | 16 +- nodepool/tests/unit/test_driver_gce.py | 2 +- nodepool/tests/unit/test_driver_ibmvpc.py | 8 +- nodepool/tests/unit/test_driver_kubernetes.py | 14 +- nodepool/tests/unit/test_driver_metastatic.py | 15 +- nodepool/tests/unit/test_driver_openshift.py | 12 +- .../tests/unit/test_driver_openshiftpods.py | 4 +- nodepool/tests/unit/test_driver_static.py | 51 ++-- nodepool/tests/unit/test_drivers.py | 2 +- nodepool/tests/unit/test_launcher.py | 267 +++++++++--------- nodepool/tests/unit/test_webapp.py | 18 +- nodepool/zk/zookeeper.py | 51 ++-- 18 files changed, 302 insertions(+), 277 deletions(-) diff --git a/nodepool/driver/statemachine.py b/nodepool/driver/statemachine.py index 84a3741ab..bd3563909 100644 --- a/nodepool/driver/statemachine.py +++ b/nodepool/driver/statemachine.py @@ -99,7 +99,7 @@ class StateMachineNodeLauncher(stats.StatsReporter): diskimage = self.provider_config.diskimages[ label.diskimage.name] cloud_image = self.zk.getMostRecentImageUpload( - diskimage.name, self.provider_config.name) + diskimage.name, self.provider_config.name, cached=True) if not cloud_image: raise exceptions.LaunchNodepoolException( @@ -374,7 +374,7 @@ class StateMachineHandler(NodeRequestHandler): else: if not self.zk.getMostRecentImageUpload( self.pool.labels[label].diskimage.name, - self.provider.name): + self.provider.name, cached=True): return False return True diff --git a/nodepool/launcher.py b/nodepool/launcher.py index 93f433bf3..c669c6b1b 100644 --- a/nodepool/launcher.py +++ b/nodepool/launcher.py @@ -1176,7 +1176,8 @@ class NodePool(threading.Thread): for pool_label in pool.labels.values(): if pool_label.diskimage: if self.zk.getMostRecentImageUpload( - pool_label.diskimage.name, pool.provider.name): + pool_label.diskimage.name, pool.provider.name, + cached=True): return True else: manager = self.getProviderManager(pool.provider.name) diff --git a/nodepool/tests/__init__.py b/nodepool/tests/__init__.py index 820ed1b61..3714ed0aa 100644 --- a/nodepool/tests/__init__.py +++ b/nodepool/tests/__init__.py @@ -460,7 +460,7 @@ class DBTestCase(BaseTestCase): os.close(fd) return path - def wait_for_config(self, pool): + def waitForConfig(self, pool): for x in range(300): if pool.config is not None: return @@ -653,6 +653,36 @@ class DBTestCase(BaseTestCase): self.addCleanup(pool.stop) return pool + def startPool(self, pool): + pool.start() + self.waitForConfig(pool) + actual_uploads = set() + for image_name in self.zk.getImageNames(): + for build_no in self.zk.getBuildNumbers(image_name): + for provider in self.zk.getBuildProviders( + image_name, build_no): + for upload_no in self.zk.getImageUploadNumbers( + image_name, build_no, provider): + upload = self.zk.getImageUpload(image_name, build_no, + provider, upload_no) + if upload and upload.state == zk.READY: + actual_uploads.add( + (upload.image_name, upload.build_id, + upload.provider_name, upload.id)) + for _ in iterate_timeout(ONE_MINUTE, Exception, + "Cache sync", + interval=1): + cached_uploads = set() + for upload in pool.zk.getCachedImageUploads(): + if upload.state == zk.READY: + cached_uploads.add((upload.image_name, upload.build_id, + upload.provider_name, upload.id)) + # In case a new image is built while we were waiting, + # just check that the cache shows at least what we saw + # at the start of the method. + if actual_uploads.issubset(cached_uploads): + break + def useWebApp(self, *args, **kwargs): app = webapp.WebApp(*args, **kwargs) self.addCleanup(app.stop) diff --git a/nodepool/tests/unit/test_builder.py b/nodepool/tests/unit/test_builder.py index 77d007ce3..66deae3a7 100644 --- a/nodepool/tests/unit/test_builder.py +++ b/nodepool/tests/unit/test_builder.py @@ -183,12 +183,12 @@ class TestNodePoolBuilder(tests.DBTestCase): get_fake_client)) configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) # NOTE(pabelanger): Disable CleanupWorker thread for nodepool-builder # as we currently race it to validate our failed uploads. self.useBuilder(configfile, cleanup_interval=0) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) diff --git a/nodepool/tests/unit/test_commands.py b/nodepool/tests/unit/test_commands.py index 4b8c79d32..f38e9ca1b 100644 --- a/nodepool/tests/unit/test_commands.py +++ b/nodepool/tests/unit/test_commands.py @@ -131,9 +131,9 @@ class TestNodepoolCMD(tests.DBTestCase): def test_list_nodes(self): configfile = self.setup_config('node.yaml') self.useBuilder(configfile) - pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.waitForNodes('fake-label') for _ in iterate_timeout(10, Exception, "assert nodes are listed"): @@ -148,9 +148,9 @@ class TestNodepoolCMD(tests.DBTestCase): def test_list_nodes_detail(self): configfile = self.setup_config('node.yaml') self.useBuilder(configfile) - pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.waitForNodes('fake-label') for _ in iterate_timeout(10, Exception, "assert nodes are listed"): try: @@ -219,7 +219,7 @@ class TestNodepoolCMD(tests.DBTestCase): configfile = self.setup_config('node_diskimage_pause.yaml') self.useBuilder(configfile) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('fake-label2') self.assertEqual(len(nodes), 1) self.assert_listed(configfile, ['dib-image-list'], 1, 'fake-image', 0) @@ -229,7 +229,7 @@ class TestNodepoolCMD(tests.DBTestCase): configfile = self.setup_config('node_image_upload_pause.yaml') self.useBuilder(configfile) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('fake-label2') self.assertEqual(len(nodes), 1) # Make sure diskimages were built. @@ -241,10 +241,10 @@ class TestNodepoolCMD(tests.DBTestCase): def test_dib_image_delete(self): configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) # Check the image exists @@ -263,10 +263,10 @@ class TestNodepoolCMD(tests.DBTestCase): # Test deletion of images without a .d manifest folder # e.g. When using a custom image creation configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) builder = self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) # Check the image exists @@ -333,10 +333,10 @@ class TestNodepoolCMD(tests.DBTestCase): def test_delete(self): configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -356,6 +356,7 @@ class TestNodepoolCMD(tests.DBTestCase): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) + self.waitForImage('fake-provider', 'fake-image') # (Shrews): This is a hack to avoid a race with the DeletedNodeWorker # thread where it may see that our direct call to NodeDeleter.delete() @@ -367,8 +368,7 @@ class TestNodepoolCMD(tests.DBTestCase): # preventing the node znode from being deleted. pool.delete_interval = 5 - pool.start() - self.waitForImage('fake-provider', 'fake-image') + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -386,10 +386,10 @@ class TestNodepoolCMD(tests.DBTestCase): def test_hold(self): configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -431,10 +431,10 @@ class TestNodepoolCMD(tests.DBTestCase): def test_attempt_hold_busy_node(self): configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -470,7 +470,7 @@ class TestNodepoolCMD(tests.DBTestCase): def test_attempt_request_held_static_node(self): configfile = self.setup_config('static-basic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -505,10 +505,10 @@ class TestNodepoolCMD(tests.DBTestCase): def test_attempt_request_held_node(self): configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -557,10 +557,10 @@ class TestNodepoolCMD(tests.DBTestCase): def test_request_list(self): configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -580,11 +580,11 @@ class TestNodepoolCMD(tests.DBTestCase): def test_info_and_erase(self): configfile = self.setup_config('info_cmd_two_provider.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() p1_image = self.waitForImage('fake-provider', 'fake-image') p2_image = self.waitForImage('fake-provider2', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) p1_nodes = self.waitForNodes('fake-label') p2_nodes = self.waitForNodes('fake-label2') @@ -645,9 +645,9 @@ class TestNodepoolCMD(tests.DBTestCase): def test_export_image_data(self): configfile = self.setup_config('node.yaml') builder = self.useBuilder(configfile) - pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.waitForNodes('fake-label') build = self.waitForBuild('fake-image', '0000000001') diff --git a/nodepool/tests/unit/test_driver_aws.py b/nodepool/tests/unit/test_driver_aws.py index 698789bb2..b61f75231 100644 --- a/nodepool/tests/unit/test_driver_aws.py +++ b/nodepool/tests/unit/test_driver_aws.py @@ -188,7 +188,7 @@ class TestDriverAws(tests.DBTestCase): # A helper method to perform a single node request configfile = self.setup_config(config_path) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -220,7 +220,7 @@ class TestDriverAws(tests.DBTestCase): # behavior. configfile = self.setup_config('aws/aws-multiple.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) reqs = [] for x in range(4): @@ -249,7 +249,7 @@ class TestDriverAws(tests.DBTestCase): # Test multiple instance type quotas (standard and high-mem) configfile = self.setup_config('aws/aws-quota.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Create a high-memory node request. req1 = zk.NodeRequest() @@ -304,7 +304,7 @@ class TestDriverAws(tests.DBTestCase): # with pool resource limits configfile = self.setup_config('aws/aws-limits.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Create a standard node request. req1 = zk.NodeRequest() @@ -349,7 +349,7 @@ class TestDriverAws(tests.DBTestCase): # with tenant resource limits configfile = self.setup_config('aws/aws-limits.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Create a high node request. req1 = zk.NodeRequest() @@ -556,7 +556,7 @@ class TestDriverAws(tests.DBTestCase): # Test dynamic tag formatting without a real node request configfile = self.setup_config('aws/aws-min-ready.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) node = self.waitForNodes('ubuntu1404-with-tags')[0] self.assertEqual(node.host_keys, ['ssh-rsa FAKEKEY']) @@ -612,7 +612,7 @@ class TestDriverAws(tests.DBTestCase): in ec2_image.tags) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -654,7 +654,7 @@ class TestDriverAws(tests.DBTestCase): in ec2_image.tags) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -743,7 +743,7 @@ class TestDriverAws(tests.DBTestCase): configfile = self.setup_config('aws/diskimage.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) for _ in iterate_timeout(30, Exception, 'instance deletion'): instance = self.ec2.Instance(instance_id) @@ -828,7 +828,7 @@ class TestDriverAws(tests.DBTestCase): configfile = self.setup_config('aws/diskimage.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) for _ in iterate_timeout(30, Exception, 'ami deletion'): image = self.ec2.Image(image_id) @@ -892,7 +892,7 @@ class TestDriverAws(tests.DBTestCase): configfile = self.setup_config('aws/diskimage.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) for _ in iterate_timeout(30, Exception, 'ami deletion'): image = self.ec2.Image(image_id) diff --git a/nodepool/tests/unit/test_driver_azure.py b/nodepool/tests/unit/test_driver_azure.py index 78f4bf585..9c63c83fa 100644 --- a/nodepool/tests/unit/test_driver_azure.py +++ b/nodepool/tests/unit/test_driver_azure.py @@ -67,7 +67,7 @@ class TestDriverAzure(tests.DBTestCase): 'azure.yaml', auth_path=self.fake_azure.auth_file.name) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.tenant_name = 'tenant-1' @@ -109,7 +109,7 @@ class TestDriverAzure(tests.DBTestCase): 'azure-min-ready.yaml', auth_path=self.fake_azure.auth_file.name) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) node = self.waitForNodes('bionic')[0] self.assertEqual(node.state, zk.READY) @@ -152,7 +152,7 @@ class TestDriverAzure(tests.DBTestCase): 'azure-diskimage.yaml', auth_path=self.fake_azure.auth_file.name) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('bionic') @@ -177,7 +177,7 @@ class TestDriverAzure(tests.DBTestCase): 'azure-external-image.yaml', auth_path=self.fake_azure.auth_file.name) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('bionic') @@ -225,7 +225,7 @@ class TestDriverAzure(tests.DBTestCase): 'azure.yaml', auth_path=self.fake_azure.auth_file.name) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('image-by-name') @@ -260,7 +260,7 @@ class TestDriverAzure(tests.DBTestCase): 'azure.yaml', auth_path=self.fake_azure.auth_file.name) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('image-by-tag') @@ -288,7 +288,7 @@ class TestDriverAzure(tests.DBTestCase): 'azure.yaml', auth_path=self.fake_azure.auth_file.name) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('windows-password') @@ -320,7 +320,7 @@ class TestDriverAzure(tests.DBTestCase): 'azure.yaml', auth_path=self.fake_azure.auth_file.name) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('windows-generate') diff --git a/nodepool/tests/unit/test_driver_gce.py b/nodepool/tests/unit/test_driver_gce.py index 67c2c5bc5..3a99d35d9 100644 --- a/nodepool/tests/unit/test_driver_gce.py +++ b/nodepool/tests/unit/test_driver_gce.py @@ -274,7 +274,7 @@ class TestDriverGce(tests.DBTestCase): def _test_gce_machine(self, pool, label, is_valid_config=True, host_key_checking=True): - pool.start() + self.startPool(pool) self._wait_for_provider(pool, 'gcloud-provider') diff --git a/nodepool/tests/unit/test_driver_ibmvpc.py b/nodepool/tests/unit/test_driver_ibmvpc.py index 88d8244ef..43bbb56c1 100644 --- a/nodepool/tests/unit/test_driver_ibmvpc.py +++ b/nodepool/tests/unit/test_driver_ibmvpc.py @@ -78,7 +78,7 @@ class TestDriverIBMVPC(tests.DBTestCase): def test_ibmvpc_cloud_image_private(self): configfile = self.setup_config('ibmvpc.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('debian-private') @@ -108,7 +108,7 @@ class TestDriverIBMVPC(tests.DBTestCase): def test_ibmvpc_cloud_image_public(self): configfile = self.setup_config('ibmvpc.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('debian-public') @@ -135,7 +135,7 @@ class TestDriverIBMVPC(tests.DBTestCase): def test_ibmvpc_cloud_image_filter(self): configfile = self.setup_config('ibmvpc.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('debian-filter') @@ -170,7 +170,7 @@ class TestDriverIBMVPC(tests.DBTestCase): self.assertEqual(image.username, 'zuul') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('fake-image') diff --git a/nodepool/tests/unit/test_driver_kubernetes.py b/nodepool/tests/unit/test_driver_kubernetes.py index cc43b8749..541eb7084 100644 --- a/nodepool/tests/unit/test_driver_kubernetes.py +++ b/nodepool/tests/unit/test_driver_kubernetes.py @@ -113,7 +113,7 @@ class TestDriverKubernetes(tests.DBTestCase): # Test a pod with default values configfile = self.setup_config('kubernetes.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.tenant_name = 'tenant-1' @@ -165,7 +165,7 @@ class TestDriverKubernetes(tests.DBTestCase): # Test a pod with lots of extra settings configfile = self.setup_config('kubernetes.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.tenant_name = 'tenant-1' @@ -223,7 +223,7 @@ class TestDriverKubernetes(tests.DBTestCase): def test_kubernetes_native(self): configfile = self.setup_config('kubernetes.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('kubernetes-namespace') @@ -251,7 +251,7 @@ class TestDriverKubernetes(tests.DBTestCase): def test_kubernetes_default_label_resources(self): configfile = self.setup_config('kubernetes-default-resources.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -366,7 +366,7 @@ class TestDriverKubernetes(tests.DBTestCase): def test_kubernetes_default_label_limits(self): configfile = self.setup_config('kubernetes-default-limits.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -462,7 +462,7 @@ class TestDriverKubernetes(tests.DBTestCase): conf = self.setup_config('kubernetes-leaked-node.yaml') pool = self.useNodepool(conf, watermark_sleep=1) pool.cleanup_interval = 1 - pool.start() + self.startPool(pool) # wait for min-ready node to be available nodes = self.waitForNodes('pod-fedora') @@ -487,7 +487,7 @@ class TestDriverKubernetes(tests.DBTestCase): def _test_kubernetes_quota(self, config, pause=True): configfile = self.setup_config(config) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Start two pods to hit max-server limit reqs = [] for _ in [1, 2]: diff --git a/nodepool/tests/unit/test_driver_metastatic.py b/nodepool/tests/unit/test_driver_metastatic.py index ef2a3e92e..f232a8c5e 100644 --- a/nodepool/tests/unit/test_driver_metastatic.py +++ b/nodepool/tests/unit/test_driver_metastatic.py @@ -83,8 +83,7 @@ class TestDriverMetastatic(tests.DBTestCase): def test_metastatic(self): configfile = self.setup_config('metastatic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter._client.create_image(name="fake-image") @@ -148,8 +147,7 @@ class TestDriverMetastatic(tests.DBTestCase): def test_metastatic_startup(self): configfile = self.setup_config('metastatic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter._client.create_image(name="fake-image") @@ -163,8 +161,7 @@ class TestDriverMetastatic(tests.DBTestCase): # Restart the provider and make sure we load data correctly pool.stop() pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter._client.create_image(name="fake-image") @@ -185,8 +182,7 @@ class TestDriverMetastatic(tests.DBTestCase): def test_metastatic_config_change(self): configfile = self.setup_config('metastatic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter._client.create_image(name="fake-image") @@ -206,8 +202,7 @@ class TestDriverMetastatic(tests.DBTestCase): # Restart the provider and make sure we load data correctly pool.stop() pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter._client.create_image(name="fake-image") diff --git a/nodepool/tests/unit/test_driver_openshift.py b/nodepool/tests/unit/test_driver_openshift.py index b2cbc277e..8d3cd64ff 100644 --- a/nodepool/tests/unit/test_driver_openshift.py +++ b/nodepool/tests/unit/test_driver_openshift.py @@ -142,7 +142,7 @@ class TestDriverOpenshift(tests.DBTestCase): # Test a pod with default values configfile = self.setup_config('openshift.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.tenant_name = 'tenant-1' @@ -197,7 +197,7 @@ class TestDriverOpenshift(tests.DBTestCase): # Test a pod with lots of extra settings configfile = self.setup_config('openshift.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.tenant_name = 'tenant-1' @@ -263,7 +263,7 @@ class TestDriverOpenshift(tests.DBTestCase): def test_openshift_default_label_resources(self): configfile = self.setup_config('openshift-default-resources.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -378,7 +378,7 @@ class TestDriverOpenshift(tests.DBTestCase): def test_openshift_default_label_limits(self): configfile = self.setup_config('openshift-default-limits.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -452,7 +452,7 @@ class TestDriverOpenshift(tests.DBTestCase): def test_openshift_pull_secret(self): configfile = self.setup_config('openshift.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('pod-fedora-secret') @@ -477,7 +477,7 @@ class TestDriverOpenshift(tests.DBTestCase): def test_openshift_native(self): configfile = self.setup_config('openshift.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('openshift-project') diff --git a/nodepool/tests/unit/test_driver_openshiftpods.py b/nodepool/tests/unit/test_driver_openshiftpods.py index 352e8f1f2..9d03868f9 100644 --- a/nodepool/tests/unit/test_driver_openshiftpods.py +++ b/nodepool/tests/unit/test_driver_openshiftpods.py @@ -87,7 +87,7 @@ class TestDriverOpenshiftPods(tests.DBTestCase): def test_openshift_pod(self): configfile = self.setup_config('openshiftpods.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('pod-fedora') @@ -118,7 +118,7 @@ class TestDriverOpenshiftPods(tests.DBTestCase): def test_openshift_pod_secrets(self): configfile = self.setup_config('openshiftpods.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED req.node_types.append('pod-fedora-secret') diff --git a/nodepool/tests/unit/test_driver_static.py b/nodepool/tests/unit/test_driver_static.py index b1581362a..0bfc027a8 100644 --- a/nodepool/tests/unit/test_driver_static.py +++ b/nodepool/tests/unit/test_driver_static.py @@ -47,7 +47,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-basic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for node pre-registration") nodes = self.waitForNodes('fake-label') @@ -76,7 +76,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-python-path.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for node pre-registration") nodes = self.waitForNodes('fake-label') @@ -97,7 +97,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-multiname.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for node pre-registration") nodes = self.waitForNodes('fake-label', 1) @@ -138,7 +138,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-unresolvable.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for node pre-registration") nodes = self.waitForNodes('fake-label') @@ -163,7 +163,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-basic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for initial node") nodes = self.waitForNodes('fake-label') @@ -183,7 +183,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-2-nodes.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for initial nodes") nodes = self.waitForNodes('fake-label', 2) @@ -204,7 +204,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-basic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for initial node") nodes = self.waitForNodes('fake-label') @@ -224,7 +224,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-parallel-increase.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for initial nodes") nodes = self.waitForNodes('fake-label', 2) @@ -244,7 +244,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-basic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for initial node") nodes = self.waitForNodes('fake-label') @@ -268,7 +268,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-basic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for initial node") nodes = self.waitForNodes('fake-label') @@ -276,7 +276,7 @@ class TestDriverStatic(tests.DBTestCase): pool.stop() configfile = self.setup_config('static-multilabel.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for new label") nodes = self.waitForNodes('fake-label2') @@ -290,7 +290,7 @@ class TestDriverStatic(tests.DBTestCase): def test_static_multilabel(self): configfile = self.setup_config('static-multilabel.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertIn('fake-label', nodes[0].type) self.assertIn('fake-label2', nodes[0].type) @@ -299,7 +299,7 @@ class TestDriverStatic(tests.DBTestCase): def test_static_handler(self): configfile = self.setup_config('static.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(nodes[0].slot, 0) self.waitForNodes('fake-concurrent-label', 2) @@ -318,7 +318,7 @@ class TestDriverStatic(tests.DBTestCase): def test_static_waiting_handler(self): configfile = self.setup_config('static-2-nodes-multilabel.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -352,7 +352,7 @@ class TestDriverStatic(tests.DBTestCase): def test_label_quota(self): configfile = self.setup_config('static-2-nodes-multilabel.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req1 = zk.NodeRequest() req1.state = zk.REQUESTED @@ -396,7 +396,7 @@ class TestDriverStatic(tests.DBTestCase): pool = self.useNodepool(configfile, watermark_sleep=1) # Make sure the cleanup worker is called that reallocated the node pool.cleanup_interval = .1 - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -418,7 +418,7 @@ class TestDriverStatic(tests.DBTestCase): def test_static_waiting_handler_order(self): configfile = self.setup_config('static-basic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -478,7 +478,7 @@ class TestDriverStatic(tests.DBTestCase): def test_static_multinode_handler(self): configfile = self.setup_config('static.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -494,9 +494,8 @@ class TestDriverStatic(tests.DBTestCase): def test_static_multiprovider_handler(self): configfile = self.setup_config('multiproviders.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) - self.wait_for_config(pool) manager = pool.getProviderManager('openstack-provider') manager.adapter._client.create_image(name="fake-image") manager.adapter.IMAGE_CHECK_TIMEOUT = 0 @@ -527,7 +526,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-basic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) self.assertEqual(nodes[0].slot, 0) @@ -559,7 +558,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-basic.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -585,7 +584,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-two-node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes1 = self.waitForNodes('fake-label') self.assertEqual(len(nodes1), 1) nodes2 = self.waitForNodes('fake-label2') @@ -639,7 +638,7 @@ class TestDriverStatic(tests.DBTestCase): configfile = self.setup_config('static-no-check.yaml') with mock.patch("nodepool.nodeutils.nodescan") as nodescan_mock: pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) nodescan_mock.assert_not_called() @@ -650,7 +649,7 @@ class TestDriverStatic(tests.DBTestCase): ''' configfile = self.setup_config('static-shell-type.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for node pre-registration") nodes = self.waitForNodes('fake-label') @@ -667,7 +666,7 @@ class TestDriverStatic(tests.DBTestCase): """Test that a missing static node is added""" configfile = self.setup_config('static-2-nodes.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for initial nodes") nodes = self.waitForNodes('fake-label', 2) diff --git a/nodepool/tests/unit/test_drivers.py b/nodepool/tests/unit/test_drivers.py index 32e5e8330..d351c5997 100644 --- a/nodepool/tests/unit/test_drivers.py +++ b/nodepool/tests/unit/test_drivers.py @@ -29,7 +29,7 @@ class TestDrivers(tests.DBTestCase): def test_external_driver_handler(self): configfile = self.setup_config('external_driver.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('test-label') self.assertEqual(len(nodes), 1) diff --git a/nodepool/tests/unit/test_launcher.py b/nodepool/tests/unit/test_launcher.py index d5102feff..7fa4a01a2 100644 --- a/nodepool/tests/unit/test_launcher.py +++ b/nodepool/tests/unit/test_launcher.py @@ -54,7 +54,7 @@ class TestLauncher(tests.DBTestCase): nodepool.launcher.LOCK_CLEANUP = 1 pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -148,7 +148,7 @@ class TestLauncher(tests.DBTestCase): self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.waitForNodes('fake-label1') self.waitForNodes('fake-label2') @@ -210,8 +210,7 @@ class TestLauncher(tests.DBTestCase): nodepool.launcher.LOCK_CLEANUP = 1 pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) client = pool.getProviderManager('fake-provider').adapter._getClient() @@ -315,8 +314,7 @@ class TestLauncher(tests.DBTestCase): nodepool.launcher.LOCK_CLEANUP = 1 pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) # wait for min-ready nodes if configured # node requests must be deferred when at tenant quota even if there @@ -469,8 +467,7 @@ class TestLauncher(tests.DBTestCase): nodepool.launcher.LOCK_CLEANUP = 1 pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) req1 = zk.NodeRequest() req1.state = zk.REQUESTED @@ -529,8 +526,7 @@ class TestLauncher(tests.DBTestCase): nodepool.launcher.LOCK_CLEANUP = 1 pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) client = pool.getProviderManager('fake-provider').adapter._getClient() @@ -589,8 +585,7 @@ class TestLauncher(tests.DBTestCase): self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter.createServer_fails = 2 @@ -613,8 +608,7 @@ class TestLauncher(tests.DBTestCase): self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -660,7 +654,7 @@ class TestLauncher(tests.DBTestCase): self.useBuilder(configfile) self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Get an initial node ready req = zk.NodeRequest() @@ -686,7 +680,7 @@ class TestLauncher(tests.DBTestCase): ''' configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -700,12 +694,12 @@ class TestLauncher(tests.DBTestCase): def test_node(self): """Test that an image and node are created""" configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) builder = self.useBuilder(configfile) - pool.start() image = self.waitForImage('fake-provider', 'fake-image') self.assertEqual(image.username, 'zuul') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) provider = (builder._upload_workers[0]._config. provider_managers['fake-provider']) cloud_image = provider.adapter._findImage(image.external_id) @@ -726,11 +720,12 @@ class TestLauncher(tests.DBTestCase): def test_node_metadata(self): """Test that node metadata is set""" configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() image = self.waitForImage('fake-provider', 'fake-image') self.assertEqual(image.username, 'zuul') + + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -749,11 +744,11 @@ class TestLauncher(tests.DBTestCase): def test_node_network_cli(self): """Same as test_node but using connection-type network_cli""" configfile = self.setup_config('node-network_cli.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() image = self.waitForImage('fake-provider', 'fake-image') self.assertEqual(image.username, 'zuul') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -767,11 +762,11 @@ class TestLauncher(tests.DBTestCase): def test_node_host_key_checking_false(self): """Test that images and nodes are created""" configfile = self.setup_config('node-host-key-checking.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() image = self.waitForImage('fake-provider', 'fake-image') self.assertEqual(image.username, 'zuul') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) label1_nodes = self.waitForNodes('fake-label') label2_nodes = self.waitForNodes('fake-label2') @@ -794,19 +789,19 @@ class TestLauncher(tests.DBTestCase): # nodepool-builder needs access to both providers to upload images configfile = self.setup_config('node_two_provider.yaml') self.useBuilder(configfile) - # Start up first launcher - configfile1 = self.setup_config('node.yaml') - pool1 = self.useNodepool(configfile1, watermark_sleep=1) - pool1.start() - # Start up second launcher - configfile2 = self.setup_config('node_second_provider.yaml') - pool2 = self.useNodepool(configfile2, watermark_sleep=1) - pool2.start() # Validate we have images in both providers image1 = self.waitForImage('fake-provider', 'fake-image') self.assertEqual(image1.username, 'zuul') image2 = self.waitForImage('fake-provider2', 'fake-image') self.assertEqual(image2.username, 'zuul') + # Start up first launcher + configfile1 = self.setup_config('node.yaml') + pool1 = self.useNodepool(configfile1, watermark_sleep=1) + self.startPool(pool1) + # Start up second launcher + configfile2 = self.setup_config('node_second_provider.yaml') + pool2 = self.useNodepool(configfile2, watermark_sleep=1) + self.startPool(pool2) # We don't need to check which provider launched the min-ready, just # that one was launched. @@ -820,11 +815,11 @@ class TestLauncher(tests.DBTestCase): """Test that a node request for a specific provider is honored""" configfile = self.setup_config('node_two_provider.yaml') self.useBuilder(configfile) - pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() # Validate we have images in both providers self.waitForImage('fake-provider', 'fake-image') self.waitForImage('fake-provider2', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.waitForNodes('fake-label', 1) req1 = zk.NodeRequest() @@ -855,11 +850,11 @@ class TestLauncher(tests.DBTestCase): """Test that a node request for a missing provider is handled""" configfile = self.setup_config('node_two_provider.yaml') self.useBuilder(configfile) - pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() # Validate we have images in both providers self.waitForImage('fake-provider', 'fake-image') self.waitForImage('fake-provider2', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.waitForNodes('fake-label', 1) req1 = zk.NodeRequest() @@ -878,9 +873,9 @@ class TestLauncher(tests.DBTestCase): when the requested labels are supported.""" configfile = self.setup_config('node.yaml') self.useBuilder(configfile) - pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.waitForNodes('fake-label', 1) # Create a dummy launcher with a different set of supported labels @@ -914,10 +909,10 @@ class TestLauncher(tests.DBTestCase): def test_node_boot_from_volume(self): """Test that an image and node are created from a volume""" configfile = self.setup_config('node_boot_from_volume.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -927,20 +922,20 @@ class TestLauncher(tests.DBTestCase): def test_disabled_label(self): """Test that a node is not created with min-ready=0""" configfile = self.setup_config('node_disabled_label.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.assertEqual([], self.zk.getNodeRequests()) self.assertEqual([], self.zk.getNodes()) def test_node_net_name(self): """Test that a node is created with proper net name""" configfile = self.setup_config('node_net_name.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) label1_nodes = self.waitForNodes('fake-label1') label2_nodes = self.waitForNodes('fake-label2') @@ -964,10 +959,10 @@ class TestLauncher(tests.DBTestCase): def test_node_security_group(self): """Test that an image and node are created with sec_group specified""" configfile = self.setup_config('node_security_group.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') nodes_def_sg = self.waitForNodes('fake-label2') self.assertEqual(len(nodes), 1) @@ -984,10 +979,10 @@ class TestLauncher(tests.DBTestCase): def test_node_flavor_name(self): """Test that a node is created with a flavor name""" configfile = self.setup_config('node_flavor_name.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) self.assertEqual(nodes[0].provider, 'fake-provider') @@ -996,10 +991,10 @@ class TestLauncher(tests.DBTestCase): def test_node_vhd_image(self): """Test that a image and node are created vhd image""" configfile = self.setup_config('node_vhd.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) self.assertEqual(nodes[0].provider, 'fake-provider') @@ -1021,10 +1016,10 @@ class TestLauncher(tests.DBTestCase): def test_dib_upload_fail(self): """Test that an image upload failure is contained.""" configfile = self.setup_config('node_upload_fail.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile, num_uploaders=2) - pool.start() self.waitForImage('fake-provider2', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label', 2) self.assertEqual(len(nodes), 2) total_nodes = sum(1 for _ in self.zk.nodeIterator()) @@ -1039,10 +1034,10 @@ class TestLauncher(tests.DBTestCase): def test_node_az(self): """Test that an image and node are created with az specified""" configfile = self.setup_config('node_az.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) self.assertEqual(nodes[0].provider, 'fake-provider') @@ -1051,11 +1046,11 @@ class TestLauncher(tests.DBTestCase): def test_node_ipv6(self): """Test that ipv6 existence either way works fine.""" configfile = self.setup_config('node_ipv6.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider1', 'fake-image') self.waitForImage('fake-provider2', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) label1_nodes = self.waitForNodes('fake-label1') label2_nodes = self.waitForNodes('fake-label2') @@ -1078,10 +1073,10 @@ class TestLauncher(tests.DBTestCase): def test_node_delete_success(self): configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) self.assertEqual(zk.READY, nodes[0].state) @@ -1102,10 +1097,10 @@ class TestLauncher(tests.DBTestCase): def test_node_delete_DELETED_success(self): """Test we treat a node in DELETING state as deleted""" configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) self.assertEqual(zk.READY, nodes[0].state) @@ -1140,13 +1135,12 @@ class TestLauncher(tests.DBTestCase): def test_node_launch_retries(self): configfile = self.setup_config('node_launch_retry.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() - self.wait_for_config(pool) + self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter.createServer_fails = 2 - self.waitForImage('fake-provider', 'fake-image') req = zk.NodeRequest() req.state = zk.REQUESTED @@ -1178,11 +1172,10 @@ class TestLauncher(tests.DBTestCase): self.zk.storeNode(znode) configfile = self.setup_config('node_launch_retry.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() - self.wait_for_config(pool) self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -1194,13 +1187,12 @@ class TestLauncher(tests.DBTestCase): def test_node_launch_retries_with_external_id(self): configfile = self.setup_config('node_launch_retry.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() - self.wait_for_config(pool) + self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter.createServer_fails_with_external_id = 2 - self.waitForImage('fake-provider', 'fake-image') # Stop the DeletedNodeWorker so we can make sure the fake znode that # is used to delete the failed servers is still around when requesting. @@ -1237,10 +1229,10 @@ class TestLauncher(tests.DBTestCase): fakeadapter.FakeAdapter, '_deleteServer', fail_delete)) configfile = self.setup_config('node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -1269,10 +1261,10 @@ class TestLauncher(tests.DBTestCase): fakeadapter.FakeAdapter, '_deleteServer', error_delete)) configfile = self.setup_config('node_delete_error.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) # request a node req = zk.NodeRequest() @@ -1300,10 +1292,10 @@ class TestLauncher(tests.DBTestCase): def test_leaked_node(self): """Test that a leaked node is deleted""" configfile = self.setup_config('leaked_node.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.log.debug("Waiting for initial pool...") nodes = self.waitForNodes('fake-label') self.log.debug("...done waiting for initial pool.") @@ -1336,10 +1328,10 @@ class TestLauncher(tests.DBTestCase): def test_max_ready_age(self): """Test a node with exceeded max-ready-age is deleted""" configfile = self.setup_config('node_max_ready_age.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.log.debug("Waiting for initial pool...") nodes = self.waitForNodes('fake-label') self.log.debug("...done waiting for initial pool.") @@ -1351,10 +1343,10 @@ class TestLauncher(tests.DBTestCase): def test_max_hold_age(self): """Test a held node with exceeded max-hold-age is deleted""" configfile = self.setup_config('node_max_hold_age.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.log.debug("Waiting for initial pool...") nodes = self.waitForNodes('fake-label') self.log.debug("...done waiting for initial pool.") @@ -1376,10 +1368,10 @@ class TestLauncher(tests.DBTestCase): """Test a held node is deleted when past its operator-specified TTL, no max-hold-age set""" configfile = self.setup_config('node_max_hold_age_no_default.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.log.debug("Waiting for initial pool...") nodes = self.waitForNodes('fake-label') self.log.debug("...done waiting for initial pool.") @@ -1402,10 +1394,10 @@ class TestLauncher(tests.DBTestCase): """Test a held node is deleted when past its operator-specified TTL, even when the type is bad""" configfile = self.setup_config('node_max_hold_age_no_default.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.log.debug("Waiting for initial pool...") nodes = self.waitForNodes('fake-label') self.log.debug("...done waiting for initial pool.") @@ -1427,10 +1419,10 @@ class TestLauncher(tests.DBTestCase): def test_hold_expiration_bad_type_coercion(self): """Test a held node uses default expiration value when type is bad""" configfile = self.setup_config('node_max_hold_age_no_default.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.log.debug("Waiting for initial pool...") nodes = self.waitForNodes('fake-label') self.log.debug("...done waiting for initial pool.") @@ -1451,10 +1443,10 @@ class TestLauncher(tests.DBTestCase): """Test a held node is deleted when past its operator-specified TTL, with max-hold-age set in the configuration""" configfile = self.setup_config('node_max_hold_age_2.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.log.debug("Waiting for initial pool...") nodes = self.waitForNodes('fake-label', 2) self.log.debug("...done waiting for initial pool.") @@ -1496,10 +1488,10 @@ class TestLauncher(tests.DBTestCase): """Test a held node is deleted after max-hold-age seconds if the operator specifies a larger TTL""" configfile = self.setup_config('node_max_hold_age_2.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) self.log.debug("Waiting for initial pool...") nodes = self.waitForNodes('fake-label', 2) self.log.debug("...done waiting for initial pool.") @@ -1549,11 +1541,11 @@ class TestLauncher(tests.DBTestCase): def test_label_provider(self): """Test that only providers listed in the label satisfy the request""" configfile = self.setup_config('node_label_provider.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') self.waitForImage('fake-provider2', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) self.assertEqual(nodes[0].provider, 'fake-provider2') @@ -1583,10 +1575,10 @@ class TestLauncher(tests.DBTestCase): (req, node) = self._create_pending_request() configfile = self.setup_config('node_lost_requests.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) self.waitForImage('fake-provider', 'fake-image') - pool.start() + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) req = self.waitForNodeRequest(req, (zk.FULFILLED,)) # Since our config file has min-ready=0, we should be able to re-use # the previously assigned node, thus making sure that the cleanup @@ -1606,7 +1598,7 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('node_lost_requests.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() + self.startPool(pool) while True: node = self.zk.getNode(node.id) @@ -1616,10 +1608,10 @@ class TestLauncher(tests.DBTestCase): def test_multiple_pools(self): """Test that an image and node are created""" configfile = self.setup_config('multiple_pools.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) lab1 = self.waitForNodes('fake-label1') lab2 = self.waitForNodes('fake-label2') @@ -1640,8 +1632,7 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('node_unmanaged_image.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter.IMAGE_CHECK_TIMEOUT = 1 manager.adapter._client.create_image(name="fake-image") @@ -1680,8 +1671,7 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('unmanaged_image_provider_name.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) manager = pool.getProviderManager('fake-provider') manager.adapter.IMAGE_CHECK_TIMEOUT = 1 manager.adapter._client.create_image(name="provider-named-image") @@ -1696,7 +1686,7 @@ class TestLauncher(tests.DBTestCase): """ configfile = self.setup_config('unmanaged_image_provider_id.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.log.debug("Waiting for node") nodes = self.waitForNodes('fake-label') self.assertEqual(len(nodes), 1) @@ -1709,7 +1699,7 @@ class TestLauncher(tests.DBTestCase): self.useBuilder(configfile) self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Create a request that uses all capacity (2 servers) req = zk.NodeRequest() @@ -1741,12 +1731,12 @@ class TestLauncher(tests.DBTestCase): def test_node_auto_floating_ip(self): """Test that auto-floating-ip option works fine.""" configfile = self.setup_config('node_auto_floating_ip.yaml') - pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() self.waitForImage('fake-provider1', 'fake-image') self.waitForImage('fake-provider2', 'fake-image') self.waitForImage('fake-provider3', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) label1_nodes = self.waitForNodes('fake-label1') label2_nodes = self.waitForNodes('fake-label2') label3_nodes = self.waitForNodes('fake-label3') @@ -1777,13 +1767,13 @@ class TestLauncher(tests.DBTestCase): """Test using secure.conf file""" configfile = self.setup_config('secure_file_config.yaml') securefile = self.setup_secure('secure_file_secure.yaml') + self.useBuilder(configfile, securefile=securefile) + image = self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool( configfile, secure_conf=securefile, watermark_sleep=1) - self.useBuilder(configfile, securefile=securefile) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) fake_image = pool.config.diskimages['fake-image'] self.assertIn('REG_PASSWORD', fake_image.env_vars) @@ -1794,7 +1784,6 @@ class TestLauncher(tests.DBTestCase): f'{self.zookeeper_chroot}') self.assertEqual(expected, zk_servers) - image = self.waitForImage('fake-provider', 'fake-image') self.assertEqual(image.username, 'zuul') nodes = self.waitForNodes('fake-label') @@ -1809,7 +1798,7 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('launcher_two_provider.yaml') self.useBuilder(configfile) pool = self.useNodepool(configfile, watermark_sleep=.5) - pool.start() + self.startPool(pool) self.waitForNodes('fake-label') self.assertEqual(2, len(pool._pool_threads)) @@ -1834,13 +1823,13 @@ class TestLauncher(tests.DBTestCase): """Test that broken provider doesn't fail node requests.""" configfile = self.setup_config('launcher_two_provider_max_1.yaml') self.useBuilder(configfile) - pool = self.useNodepool(configfile, watermark_sleep=.5) - pool.start() - self.wait_for_config(pool) - # Steady state at images available. self.waitForImage('fake-provider', 'fake-image') self.waitForImage('fake-provider2', 'fake-image') + + pool = self.useNodepool(configfile, watermark_sleep=.5) + self.startPool(pool) + # We have now reached steady state and can manipulate the system to # test failing cloud behavior. @@ -1959,7 +1948,7 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('disabled_provider.yaml') self.useBuilder(configfile) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -1978,8 +1967,7 @@ class TestLauncher(tests.DBTestCase): ''' configfile = self.setup_config('broken_provider_config.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) manager = pool.getProviderManager('good-provider') manager.adapter._client.create_image(name="good-image") @@ -2011,7 +1999,7 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('wedge_test.yaml') self.useBuilder(configfile) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Wait for fake-label1 min-ready request to be fulfilled, which will # put us at maximum capacity with max-servers of 1. @@ -2041,7 +2029,7 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('launcher_reg1.yaml') self.useBuilder(configfile) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.waitForNodes('fake-label') launcher_pools = self.zk.getRegisteredPools() @@ -2070,10 +2058,10 @@ class TestLauncher(tests.DBTestCase): # use a config with min-ready of 0 configfile = self.setup_config('node_launch_retry.yaml') self.useBuilder(configfile) + self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) pool.cleanup_interval = 60 - pool.start() - self.waitForImage('fake-provider', 'fake-image') + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -2107,10 +2095,10 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('node_launch_retry.yaml') self.useBuilder(configfile) + self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) pool.cleanup_interval = 60 - pool.start() - self.waitForImage('fake-provider', 'fake-image') + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -2147,9 +2135,9 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('node_launch_retry.yaml') self.useBuilder(configfile) - pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() self.waitForImage('fake-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) req = zk.NodeRequest() req.state = zk.REQUESTED @@ -2203,7 +2191,7 @@ class TestLauncher(tests.DBTestCase): pool = self.useNodepool(configfile, watermark_sleep=1) pool.cleanup_interval = 60 - pool.start() + self.startPool(pool) # Wait for request handling to occur while not mock_poll.call_count: @@ -2240,7 +2228,7 @@ class TestLauncher(tests.DBTestCase): self.useBuilder(configfile) self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Create a request that uses all capacity (2 servers) req = zk.NodeRequest() @@ -2302,7 +2290,7 @@ class TestLauncher(tests.DBTestCase): self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Create a request with ignore-provider-quota set to false that should # fail because it will decline the request because "it would exceed @@ -2337,7 +2325,7 @@ class TestLauncher(tests.DBTestCase): self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Create a request with ignore-provider-quota set to true that should # pass regardless of the lack of cloud/provider quota. @@ -2434,7 +2422,7 @@ class TestLauncher(tests.DBTestCase): self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) # Create a request with ignore-provider-quota set to true that should # pass regardless of the lack of cloud/provider quota. @@ -2487,7 +2475,7 @@ class TestLauncher(tests.DBTestCase): self.waitForImage('fake-provider', 'fake-image') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) self.waitForAnyNodeInState(zk.ABORTED) @@ -2533,7 +2521,7 @@ class TestLauncher(tests.DBTestCase): self.zk.storeNodeRequest(req2) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) req2 = self.waitForNodeRequest(req2) self.assertEqual(req2.state, zk.FULFILLED) @@ -2553,8 +2541,7 @@ class TestLauncher(tests.DBTestCase): self.assertEqual(image.username, 'fake-username') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() - self.wait_for_config(pool) + self.startPool(pool) # Request with a higher relative priority coming in first, but # requesting a label that is not available. @@ -2596,7 +2583,7 @@ class TestLauncher(tests.DBTestCase): pool = self.useNodepool(configfile, watermark_sleep=1) pool.cleanup_interval = .1 - pool.start() + self.startPool(pool) while self.zk.kazoo_client.exists(path): time.sleep(.1) @@ -2606,7 +2593,7 @@ class TestLauncher(tests.DBTestCase): self.useBuilder(configfile) pool = self.useNodepool(configfile, watermark_sleep=1) pool.cleanup_interval = 1 - pool.start() + self.startPool(pool) self.waitForNodes('fake-label') manager = pool.getProviderManager('fake-provider') @@ -2643,7 +2630,7 @@ class TestLauncher(tests.DBTestCase): configfile = self.setup_config('node.yaml') self.useBuilder(configfile) pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) nodes = self.waitForNodes('fake-label') self.assertEqual(1, len(nodes)) @@ -2670,10 +2657,10 @@ class TestLauncher(tests.DBTestCase): """Test provider priorities""" configfile = self.setup_config('priority.yaml') self.useBuilder(configfile) - pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() self.waitForImage('low-provider', 'fake-image') self.waitForImage('high-provider', 'fake-image') + pool = self.useNodepool(configfile, watermark_sleep=1) + self.startPool(pool) # The first request should be handled by the highest priority # provider (high-provider; priority 1) diff --git a/nodepool/tests/unit/test_webapp.py b/nodepool/tests/unit/test_webapp.py index 8c575fb1c..bcb01d6e4 100644 --- a/nodepool/tests/unit/test_webapp.py +++ b/nodepool/tests/unit/test_webapp.py @@ -34,7 +34,7 @@ class TestWebApp(tests.DBTestCase): def test_image_list(self): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) webapp = self.useWebApp(pool, port=0) webapp.start() port = webapp.server.socket.getsockname()[1] @@ -71,7 +71,7 @@ class TestWebApp(tests.DBTestCase): def test_image_list_filtered(self): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) webapp = self.useWebApp(pool, port=0) webapp.start() port = webapp.server.socket.getsockname()[1] @@ -96,7 +96,7 @@ class TestWebApp(tests.DBTestCase): def test_image_list_json(self): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) webapp = self.useWebApp(pool, port=0) webapp.start() port = webapp.server.socket.getsockname()[1] @@ -124,7 +124,7 @@ class TestWebApp(tests.DBTestCase): def test_dib_image_list_json(self): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) webapp = self.useWebApp(pool, port=0) webapp.start() port = webapp.server.socket.getsockname()[1] @@ -154,7 +154,7 @@ class TestWebApp(tests.DBTestCase): configfile = self.setup_config("node.yaml") pool = self.useNodepool(configfile, watermark_sleep=1) - pool.start() + self.startPool(pool) webapp = self.useWebApp(pool, port=0) webapp.start() port = webapp.server.socket.getsockname()[1] @@ -209,7 +209,7 @@ class TestWebApp(tests.DBTestCase): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() + self.startPool(pool) webapp = self.useWebApp(pool, port=0) webapp.start() port = webapp.server.socket.getsockname()[1] @@ -269,7 +269,7 @@ class TestWebApp(tests.DBTestCase): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() + self.startPool(pool) webapp = self.useWebApp(pool, port=0) webapp.start() port = webapp.server.socket.getsockname()[1] @@ -305,7 +305,7 @@ class TestWebApp(tests.DBTestCase): configfile = self.setup_config('node.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) self.useBuilder(configfile) - pool.start() + self.startPool(pool) webapp = self.useWebApp(pool, port=0) webapp.start() port = webapp.server.socket.getsockname()[1] @@ -343,7 +343,7 @@ class TestWebApp(tests.DBTestCase): with self.assertRaises(HTTPError, request.urlopen, req): pass - pool.start() + self.startPool(pool) # Now wait until we get a valid response. for _ in iterate_timeout(30, Exception, 'ready succeeds'): diff --git a/nodepool/zk/zookeeper.py b/nodepool/zk/zookeeper.py index bdbe1ddb1..c0ff2addd 100644 --- a/nodepool/zk/zookeeper.py +++ b/nodepool/zk/zookeeper.py @@ -1835,39 +1835,52 @@ class ZooKeeper(ZooKeeperBase): return uploads[:count] def getMostRecentImageUpload(self, image, provider, - state=READY): + state=READY, cached=False): ''' Retrieve the most recent image upload data with the given state. :param str image: The image name. :param str provider: The provider name owning the image. :param str state: The image upload state to match on. + :param bool cached: Whether to use cached data. :returns: An ImageUpload object matching the given state, or None if there is no recent upload. ''' + uploads = [] - recent_data = None - for build_number in self.getBuildNumbers(image): - path = self._imageUploadPath(image, build_number, provider) + if cached: + uploads = self.getCachedImageUploads() + else: + for build_number in self.getBuildNumbers(image): + path = self._imageUploadPath(image, build_number, provider) + try: + upload_numbers = self.kazoo_client.get_children(path) + except kze.NoNodeError: + upload_numbers = [] - try: - uploads = self.kazoo_client.get_children(path) - except kze.NoNodeError: - uploads = [] + for upload_number in upload_numbers: + if upload_number == 'lock': # skip the upload lock node + continue + data = self.getImageUpload( + image, build_number, provider, upload_number) + if not data or data.state != state: + continue + uploads.append(data) - for upload in uploads: - if upload == 'lock': # skip the upload lock node - continue - data = self.getImageUpload( - image, build_number, provider, upload) - if not data or data.state != state: - continue - elif (recent_data is None or - recent_data.state_time < data.state_time): - recent_data = data + recent_upload = None + for upload in uploads: + if upload.image_name != image: + continue + if upload.provider_name != provider: + continue + if upload.state != state: + continue + if (recent_upload is None or + recent_upload.state_time < upload.state_time): + recent_upload = upload - return recent_data + return recent_upload def getCachedImageUploads(self): '''