Browse Source

Merge "Overriding max. starting builds."

tags/3.10.2
Zuul 1 week ago
parent
commit
6551f35c35

+ 14
- 0
doc/source/admin/components.rst View File

@@ -688,6 +688,20 @@ The following sections of ``zuul.conf`` are used by the executor:
688 688
       The executor will observe system load and determine whether
689 689
       to accept more jobs every 30 seconds.
690 690
 
691
+   .. attr:: max_starting_builds
692
+      :default: None
693
+
694
+      An executor is accepting up to as many starting builds as defined by the
695
+      :attr:`executor.load_multiplier` on systems with more than four CPU cores,
696
+      and up to twice as many on systems with four or less CPU cores. For
697
+      example, on a system with two CPUs: 2 * 2.5 * 2 - up to ten starting
698
+      builds may run on such executor; on systems with eight CPUs: 2.5 * 8 - up
699
+      to twenty starting builds may run on such executor.
700
+
701
+      On systems with high CPU/vCPU count an executor may accept too many
702
+      starting builds. This can be overwritten using this option providing a
703
+      fixed number of maximum starting builds on an executor.
704
+
691 705
    .. attr:: min_avail_hdd
692 706
       :default: 5.0
693 707
 

+ 5
- 0
releasenotes/notes/max-starting-builds-config-2a127fbfb46460c5.yaml View File

@@ -0,0 +1,5 @@
1
+---
2
+features:
3
+  - |
4
+    The maximum starting builds depending on cpu cores can be limited using
5
+    `executor.max_starting_builds` configuration.

+ 53
- 1
tests/unit/test_executor.py View File

@@ -462,6 +462,58 @@ class TestExecutorHostname(ZuulTestCase):
462 462
                          self.executor_server.hostname)
463 463
 
464 464
 
465
+class TestStartingBuildsSensor(ZuulTestCase):
466
+    config_file = 'zuul.conf'
467
+    tenant_config_file = 'config/governor/main.yaml'
468
+
469
+    def test_default_case(self):
470
+        # Given
471
+        cores = multiprocessing.cpu_count()
472
+
473
+        # When
474
+        sensor = StartingBuildsSensor(None, cores * 2.5, None)
475
+
476
+        # Then
477
+        self.assertEqual(sensor.max_starting_builds, int(cores * 2.5 * 2))
478
+        self.assertEqual(sensor.min_starting_builds, max(int(cores / 2), 1))
479
+
480
+    def test_configuration_not_exists(self):
481
+        # Given
482
+        cores = multiprocessing.cpu_count()
483
+
484
+        # When
485
+        sensor = StartingBuildsSensor(None, cores * 2.5, self.config)
486
+
487
+        # Then
488
+        self.assertEqual(sensor.max_starting_builds, int(cores * 2.5 * 2))
489
+        self.assertEqual(sensor.min_starting_builds, max(int(cores / 2), 1))
490
+
491
+    def test_configuration_override(self):
492
+        # Given
493
+        cores = multiprocessing.cpu_count()
494
+        self.config.set('executor', 'max_starting_builds', '5')
495
+
496
+        # When
497
+        sensor = StartingBuildsSensor(None, cores * 2.5, self.config)
498
+
499
+        # Then
500
+        self.assertEqual(sensor.max_starting_builds, 5)
501
+        self.assertEqual(sensor.min_starting_builds, min(
502
+            max(int(cores / 2), 1), sensor.max_starting_builds))
503
+
504
+    def test_configuration_override_affecting_min(self):
505
+        # Given
506
+        cores = multiprocessing.cpu_count()
507
+        self.config.set('executor', 'max_starting_builds', '1')
508
+
509
+        # When
510
+        sensor = StartingBuildsSensor(None, cores * 2.5, self.config)
511
+
512
+        # Then
513
+        self.assertEqual(sensor.max_starting_builds, 1)
514
+        self.assertEqual(sensor.min_starting_builds, 1)
515
+
516
+
465 517
 class TestGovernor(ZuulTestCase):
466 518
     config_file = 'zuul-executor-hostname.conf'
467 519
     tenant_config_file = 'config/governor/main.yaml'
@@ -576,7 +628,7 @@ class TestGovernor(ZuulTestCase):
576 628
         self.log.debug("Waiting for %s to start", jobname)
577 629
         timeout = time.time() + 30
578 630
         build = None
579
-        while (time.time() < timeout and not build):
631
+        while time.time() < timeout and not build:
580 632
             for b in self.builds:
581 633
                 if b.name == jobname:
582 634
                     build = b

+ 10
- 3
zuul/executor/sensors/startingbuilds.py View File

@@ -16,15 +16,22 @@ import logging
16 16
 import multiprocessing
17 17
 
18 18
 from zuul.executor.sensors import SensorInterface
19
+from zuul.lib.config import get_default
19 20
 
20 21
 
21 22
 class StartingBuildsSensor(SensorInterface):
22 23
     log = logging.getLogger("zuul.executor.sensor.startingbuilds")
23 24
 
24
-    def __init__(self, executor, max_load_avg):
25
+    def __init__(self, executor, max_load_avg, config=None):
25 26
         self.executor = executor
26
-        self.max_starting_builds = max_load_avg * 2
27
-        self.min_starting_builds = max(int(multiprocessing.cpu_count() / 2), 1)
27
+
28
+        self.max_starting_builds = get_default(
29
+            config, 'executor', 'max_starting_builds', int(max_load_avg * 2)) \
30
+            if config is not None else int(max_load_avg * 2)
31
+
32
+        self.min_starting_builds = min(
33
+            max(int(multiprocessing.cpu_count() / 2), 1),
34
+            self.max_starting_builds)
28 35
 
29 36
     def _getStartingBuilds(self):
30 37
         starting_builds = 0

+ 1
- 1
zuul/executor/server.py View File

@@ -2394,7 +2394,7 @@ class ExecutorServer(object):
2394 2394
             HDDSensor(config),
2395 2395
             self.pause_sensor,
2396 2396
             RAMSensor(config),
2397
-            StartingBuildsSensor(self, cpu_sensor.max_load_avg)
2397
+            StartingBuildsSensor(self, cpu_sensor.max_load_avg, config)
2398 2398
         ]
2399 2399
 
2400 2400
         manage_ansible = get_default(

Loading…
Cancel
Save