From e1a69af50c98490cded0b1a568e0789fbd80103d Mon Sep 17 00:00:00 2001 From: Simon Westphahl Date: Wed, 5 Mar 2025 15:06:11 +0100 Subject: [PATCH] Enable node request fallback with zuul-launcher So far we only supported Nodepool fallback labels. Change handler to also issue nodeset requests if the label is available. Change-Id: Ib7de75c5e1a7a3468d81ee0995209e13b118d57d --- .../layouts/launcher-nodeset-fallback.yaml | 114 ++++++++++++++++++ tests/unit/test_launcher.py | 38 ++++++ zuul/manager/__init__.py | 13 +- 3 files changed, 160 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/layouts/launcher-nodeset-fallback.yaml diff --git a/tests/fixtures/layouts/launcher-nodeset-fallback.yaml b/tests/fixtures/layouts/launcher-nodeset-fallback.yaml new file mode 100644 index 0000000000..f0f0482294 --- /dev/null +++ b/tests/fixtures/layouts/launcher-nodeset-fallback.yaml @@ -0,0 +1,114 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + +- nodeset: + name: debian-nodeset + alternatives: + - nodes: + label: debian-invalid + name: node + - nodes: + label: debian-normal + name: node + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: check-job + run: playbooks/check.yaml + nodeset: debian-nodeset + +- project: + name: org/project + check: + jobs: + - check-job + gate: + jobs: + - check-job + +- image: + name: debian + type: cloud + +- flavor: + name: normal + +- flavor: + name: invalid + +- label: + name: debian-normal + image: debian + flavor: normal + +- label: + name: debian-invalid + image: debian + flavor: invalid + +- section: + name: aws-base + abstract: true + connection: aws + host-key-checking: false + boot-timeout: 300 + launch-timeout: 600 + launch-attempts: 2 + object-storage: + bucket-name: zuul + key-name: zuul + flavors: + - name: normal + instance-type: t3.medium + - name: invalid + instance-type: invalid + images: + - name: debian + image-id: ami-1e749f67 + +- section: + name: aws-us-east-1 + parent: aws-base + region: us-east-1 + +- provider: + name: aws-us-east-1-main + section: aws-us-east-1 + labels: + - name: debian-normal + - name: debian-invalid diff --git a/tests/unit/test_launcher.py b/tests/unit/test_launcher.py index abeb65b7a7..005aa99e80 100644 --- a/tests/unit/test_launcher.py +++ b/tests/unit/test_launcher.py @@ -14,6 +14,7 @@ # under the License. import math +import os import textwrap import time from collections import defaultdict @@ -26,6 +27,7 @@ from zuul.launcher.client import LauncherClient import responses import testtools +import yaml from kazoo.exceptions import NoNodeError from moto import mock_aws import boto3 @@ -785,6 +787,42 @@ class TestLauncher(LauncherBaseTestCase): except NoNodeError: break + @simple_layout('layouts/launcher-nodeset-fallback.yaml', + enable_nodepool=True) + @okay_tracebacks('_getQuotaForInstanceType') + def test_nodeset_fallback(self): + # Test that nodeset fallback works + self.executor_server.hold_jobs_in_build = True + + tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') + job = tenant.layout.getJob('check-job') + alts = job.flattenNodesetAlternatives(tenant.layout) + self.assertEqual(2, len(alts)) + self.assertEqual('debian-invalid', alts[0].nodes[('node',)].label) + self.assertEqual('debian-normal', alts[1].nodes[('node',)].label) + + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + build = self.getBuildByName('check-job') + inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml') + with open(inv_path, 'r') as f: + inventory = yaml.safe_load(f) + label = inventory['all']['hosts']['node']['nodepool']['label'] + self.assertEqual('debian-normal', label) + + self.executor_server.hold_jobs_in_build = False + self.executor_server.release() + self.waitUntilSettled() + + self.assertEqual(A.data['status'], 'NEW') + self.assertEqual(A.reported, 1) + self.assertNotIn('NODE_FAILURE', A.messages[0]) + self.assertHistory([ + dict(name='check-job', result='SUCCESS', changes='1,1'), + ], ordered=False) + @simple_layout('layouts/nodepool.yaml', enable_nodepool=True) @mock.patch( 'zuul.driver.aws.awsendpoint.AwsProviderEndpoint._createInstance', diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py index 5840222584..911aa11dd0 100644 --- a/zuul/manager/__init__.py +++ b/zuul/manager/__init__.py @@ -1162,8 +1162,7 @@ class PipelineManager(metaclass=ABCMeta): log.debug("Falling back to Nodepool due to missing labels: %s", labels) return True - def _makeNodepoolRequest(self, log, build_set, job, relative_priority, - alternative=0): + def _makeNodepoolRequest(self, log, build_set, job, relative_priority): provider = self._getPausedParentProvider(build_set, job) priority = self._calculateNodeRequestPriority(build_set, job) tenant_name = build_set.item.pipeline.tenant.name @@ -1183,7 +1182,7 @@ class PipelineManager(metaclass=ABCMeta): else: job.setWaitingStatus(f'node request: {req.id}') - def _makeLauncherRequest(self, log, build_set, job, alternative=0): + def _makeLauncherRequest(self, log, build_set, job, relative_priority): provider = self._getPausedParentProvider(build_set, job) priority = self._calculateNodeRequestPriority(build_set, job) item = build_set.item @@ -2267,8 +2266,12 @@ class PipelineManager(metaclass=ABCMeta): build_set.item.getChangeForJob(job)) else: relative_priority = 0 - log = build_set.item.annotateLogger(self.log) - self._makeNodepoolRequest(log, build_set, job, relative_priority) + if self._useNodepoolFallback(log, job): + self._makeNodepoolRequest( + log, build_set, job, relative_priority) + else: + self._makeLauncherRequest( + log, build_set, job, relative_priority) return True def onNodesProvisioned(self, request, nodeset_info, build_set):