Add support for configuring executor zones

This change adds support for configuring an executor-zone for labels.

Change-Id: Icd1f23a50478e3fb822b830e9b9710e6aea0a1cb

tests/fixtures/layouts/nodepool-executor-zone.yaml

@@ -0,0 +1,171 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- pipeline:
+    name: gate
+    manager: dependent
+    success-message: Build succeeded (gate).
+    trigger:
+      gerrit:
+        - event: comment-added
+          approval:
+            - Approved: 1
+    success:
+      gerrit:
+        Verified: 2
+        submit: true
+    failure:
+      gerrit:
+        Verified: -2
+    start:
+      gerrit:
+        Verified: 0
+    precedence: high
+
+- pipeline:
+    name: post
+    manager: independent
+    trigger:
+      gerrit:
+        - event: ref-updated
+          ref: ^(?!refs/).*$
+
+- pipeline:
+    name: tag
+    manager: independent
+    trigger:
+      gerrit:
+        - event: ref-updated
+          ref: ^refs/tags/.*$
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+    nodeset:
+      nodes:
+        - label: debian-normal
+          name: controller
+
+- job:
+    name: check-job
+    run: playbooks/check.yaml
+
+- job:
+    name: post-job
+    run: playbooks/post.yaml
+
+- job:
+    name: tag-job
+    run: playbooks/tag.yaml
+
+- project:
+    name: org/project
+    check:
+      jobs:
+        - check-job
+    gate:
+      jobs:
+        - check-job
+    post:
+      jobs:
+        - post-job
+    tag:
+      jobs:
+        - tag-job
+
+- image:
+    name: debian
+    type: cloud
+
+- flavor:
+    name: normal
+
+- flavor:
+    name: large
+
+- flavor:
+    name: dedicated
+
+- flavor:
+    name: invalid
+
+- label:
+    name: debian-normal
+    image: debian
+    flavor: normal
+
+- label:
+    name: debian-large
+    image: debian
+    flavor: large
+
+- label:
+    name: debian-dedicated
+    image: debian
+    flavor: dedicated
+
+- label:
+    name: debian-invalid
+    image: debian
+    flavor: invalid
+
+- section:
+    name: aws-base
+    abstract: true
+    connection: aws
+    host-key-checking: false
+    boot-timeout: 300
+    launch-timeout: 600
+    launch-attempts: 2
+    object-storage:
+      bucket-name: zuul
+    key-name: zuul
+    flavors:
+      - name: normal
+        instance-type: t3.medium
+        volume-type: gp3
+        volume-size: 40
+        iops: 500
+        throughput: 200
+      - name: large
+        instance-type: t3.large
+      - name: dedicated
+        instance-type: t3.large
+        dedicated-host: True
+      - name: invalid
+        instance-type: invalid
+    images:
+      - name: debian
+        image-id: ami-1e749f67
+
+- section:
+    name: aws-us-east-1
+    parent: aws-base
+    region: us-east-1
+
+- provider:
+    name: aws-us-east-1-main
+    section: aws-us-east-1
+    executor-zone: us-east-1
+    labels:
+      - name: debian-normal
+        volume-type: gp3
+        volume-size: 40
+        # Change iops and omit throughput to test label overriding the
+        # flavor
+        iops: 1000
+      - name: debian-large
+      - name: debian-dedicated
+        az: us-east-1b
+      - name: debian-invalid

tests/unit/test_launcher.py

@@ -1506,3 +1506,36 @@ class TestNodesetRequestPriority(LauncherBaseTestCase):
         self.hold_nodeset_requests_in_queue = False
         self.releaseNodesetRequests(*reqs)
         self.waitUntilSettled()
+
+
+class TestExecutorZone(LauncherBaseTestCase):
+    tenant_config_file = 'config/single-tenant/main.yaml'
+
+    def setup_config(self, config_file: str):
+        config = super().setup_config(config_file)
+        config.set('executor', 'zone', 'us-east-1')
+        return config
+
+    @simple_layout('layouts/nodepool-executor-zone.yaml', enable_nodepool=True)
+    def test_jobs_executed(self):
+        self.hold_jobs_in_queue = True
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        A.addApproval('Code-Review', 2)
+        self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+        self.waitUntilSettled()
+
+        queue = list(self.executor_api.queued())
+        self.assertEqual(len(self.builds), 0)
+        self.assertEqual(len(queue), 1)
+        self.assertEqual('us-east-1', queue[0].zone)
+
+        self.hold_jobs_in_queue = False
+        self.executor_api.release()
+        self.waitUntilSettled()
+
+        self.assertEqual(self.getJobFromHistory('check-job').result,
+                         'SUCCESS')
+        self.assertEqual(A.data['status'], 'MERGED')
+        self.assertEqual(A.reported, 2)
+        self.assertEqual(self.getJobFromHistory('check-job').node,
+                         'debian-normal')

zuul/launcher/client.py

@@ -97,7 +97,9 @@ class LauncherClient:
 
     def getNodesetInfo(self, request):
         # TODO: populated other nodeset info fields
-        return NodesetInfo(nodes=list(request.nodes))
+        return NodesetInfo(
+            nodes=list(request.nodes),
+            zone=request.executor_zones[0])
 
     def acceptNodeset(self, request, nodeset):
         log = get_annotated_logger(self.log, request)

zuul/launcher/server.py

@@ -992,6 +992,7 @@ class Launcher:
             max_ready_age=label.max_ready_age,
             host_key_checking=label.host_key_checking,
             boot_timeout=label.boot_timeout,
+            executor_zone=label.executor_zone,
             request_id=request.uuid,
             zuul_event_id=request.zuul_event_id,
             connection_name=provider.connection_name,
@@ -1028,8 +1029,7 @@ class Launcher:
                         label, request, provider, log, ctx)
                     with request.activeContext(ctx):
                         request.updateProviderNode(
-                            i,
-                            uuid=node.uuid,
+                            i, node,
                             add_failed_provider=add_failed_provider,
                         )
 
@@ -1306,6 +1306,7 @@ class Launcher:
                     max_ready_age=label.max_ready_age,
                     host_key_checking=label.host_key_checking,
                     boot_timeout=label.boot_timeout,
+                    executor_zone=label.executor_zone,
                     request_id=None,
                     connection_name=provider.connection_name,
                     zuul_event_id=uuid.uuid4().hex,

zuul/model.py

@@ -2446,15 +2446,17 @@ class NodesetRequest(zkobject.LockableZKObject):
     def addProviderNode(self, provider_node):
         self.provider_node_data.append(dict(
             uuid=provider_node.uuid,
+            executor_zone=provider_node.executor_zone,
             failed_providers=[],
         ))
 
-    def updateProviderNode(self, index,
-                           uuid=None,
+    def updateProviderNode(self, index, provider_node,
                            add_failed_provider=None):
         data = self.provider_node_data[index]
-        if uuid is not None:
-            data['uuid'] = uuid
+        data.update(dict(
+            uuid=provider_node.uuid,
+            executor_zone=provider_node.executor_zone,
+        ))
         if add_failed_provider is not None:
             data['failed_providers'].append(add_failed_provider)
 
@@ -2471,6 +2473,10 @@ class NodesetRequest(zkobject.LockableZKObject):
     def nodes(self):
         return [n['uuid'] for n in self.provider_node_data]
 
+    @property
+    def executor_zones(self):
+        return [n.get('executor_zone') for n in self.provider_node_data]
+
     @property
     def created_time(self):
         return self.request_time
@@ -2612,8 +2618,9 @@ class ProviderNode(zkobject.PolymorphicZKObjectMixin,
             create_state={},
             delete_state={},
             host_key_checking=None,
-            boot_timeout=None,
             # Node data
+            boot_timeout=None,
+            executor_zone=None,
             host_id=None,
             interface_ip=None,
             public_ipv4=None,
@@ -2714,6 +2721,7 @@ class ProviderNode(zkobject.PolymorphicZKObjectMixin,
             attributes=self.attributes,
             az=self.az,
             boot_timeout=self.boot_timeout,
+            executor_zone=self.executor_zone,
             cloud=self.cloud,
             connection_port=self.connection_port,
             connection_type=self.connection_type,

zuul/provider/schema.py

@@ -27,6 +27,7 @@ from zuul.lib.voluputil import Required, Optional, Nullable
 # label.
 common_label = vs.Schema({
     Optional('boot-timeout', default=300): int,
+    Optional('executor-zone'): Nullable(str),
 })
 
 # The label attributes that can appear in a section/provider label or