Browse Source

Merge "Overriding max. starting builds."

tags/3.10.2
Zuul 5 months ago
parent
commit
6551f35c35
5 changed files with 83 additions and 5 deletions
  1. +14
    -0
      doc/source/admin/components.rst
  2. +5
    -0
      releasenotes/notes/max-starting-builds-config-2a127fbfb46460c5.yaml
  3. +53
    -1
      tests/unit/test_executor.py
  4. +10
    -3
      zuul/executor/sensors/startingbuilds.py
  5. +1
    -1
      zuul/executor/server.py

+ 14
- 0
doc/source/admin/components.rst View File

@@ -688,6 +688,20 @@ The following sections of ``zuul.conf`` are used by the executor:
The executor will observe system load and determine whether
to accept more jobs every 30 seconds.

.. attr:: max_starting_builds
:default: None

An executor is accepting up to as many starting builds as defined by the
:attr:`executor.load_multiplier` on systems with more than four CPU cores,
and up to twice as many on systems with four or less CPU cores. For
example, on a system with two CPUs: 2 * 2.5 * 2 - up to ten starting
builds may run on such executor; on systems with eight CPUs: 2.5 * 8 - up
to twenty starting builds may run on such executor.

On systems with high CPU/vCPU count an executor may accept too many
starting builds. This can be overwritten using this option providing a
fixed number of maximum starting builds on an executor.

.. attr:: min_avail_hdd
:default: 5.0


+ 5
- 0
releasenotes/notes/max-starting-builds-config-2a127fbfb46460c5.yaml View File

@@ -0,0 +1,5 @@
---
features:
- |
The maximum starting builds depending on cpu cores can be limited using
`executor.max_starting_builds` configuration.

+ 53
- 1
tests/unit/test_executor.py View File

@@ -462,6 +462,58 @@ class TestExecutorHostname(ZuulTestCase):
self.executor_server.hostname)


class TestStartingBuildsSensor(ZuulTestCase):
config_file = 'zuul.conf'
tenant_config_file = 'config/governor/main.yaml'

def test_default_case(self):
# Given
cores = multiprocessing.cpu_count()

# When
sensor = StartingBuildsSensor(None, cores * 2.5, None)

# Then
self.assertEqual(sensor.max_starting_builds, int(cores * 2.5 * 2))
self.assertEqual(sensor.min_starting_builds, max(int(cores / 2), 1))

def test_configuration_not_exists(self):
# Given
cores = multiprocessing.cpu_count()

# When
sensor = StartingBuildsSensor(None, cores * 2.5, self.config)

# Then
self.assertEqual(sensor.max_starting_builds, int(cores * 2.5 * 2))
self.assertEqual(sensor.min_starting_builds, max(int(cores / 2), 1))

def test_configuration_override(self):
# Given
cores = multiprocessing.cpu_count()
self.config.set('executor', 'max_starting_builds', '5')

# When
sensor = StartingBuildsSensor(None, cores * 2.5, self.config)

# Then
self.assertEqual(sensor.max_starting_builds, 5)
self.assertEqual(sensor.min_starting_builds, min(
max(int(cores / 2), 1), sensor.max_starting_builds))

def test_configuration_override_affecting_min(self):
# Given
cores = multiprocessing.cpu_count()
self.config.set('executor', 'max_starting_builds', '1')

# When
sensor = StartingBuildsSensor(None, cores * 2.5, self.config)

# Then
self.assertEqual(sensor.max_starting_builds, 1)
self.assertEqual(sensor.min_starting_builds, 1)


class TestGovernor(ZuulTestCase):
config_file = 'zuul-executor-hostname.conf'
tenant_config_file = 'config/governor/main.yaml'
@@ -576,7 +628,7 @@ class TestGovernor(ZuulTestCase):
self.log.debug("Waiting for %s to start", jobname)
timeout = time.time() + 30
build = None
while (time.time() < timeout and not build):
while time.time() < timeout and not build:
for b in self.builds:
if b.name == jobname:
build = b

+ 10
- 3
zuul/executor/sensors/startingbuilds.py View File

@@ -16,15 +16,22 @@ import logging
import multiprocessing

from zuul.executor.sensors import SensorInterface
from zuul.lib.config import get_default


class StartingBuildsSensor(SensorInterface):
log = logging.getLogger("zuul.executor.sensor.startingbuilds")

def __init__(self, executor, max_load_avg):
def __init__(self, executor, max_load_avg, config=None):
self.executor = executor
self.max_starting_builds = max_load_avg * 2
self.min_starting_builds = max(int(multiprocessing.cpu_count() / 2), 1)

self.max_starting_builds = get_default(
config, 'executor', 'max_starting_builds', int(max_load_avg * 2)) \
if config is not None else int(max_load_avg * 2)

self.min_starting_builds = min(
max(int(multiprocessing.cpu_count() / 2), 1),
self.max_starting_builds)

def _getStartingBuilds(self):
starting_builds = 0

+ 1
- 1
zuul/executor/server.py View File

@@ -2394,7 +2394,7 @@ class ExecutorServer(object):
HDDSensor(config),
self.pause_sensor,
RAMSensor(config),
StartingBuildsSensor(self, cpu_sensor.max_load_avg)
StartingBuildsSensor(self, cpu_sensor.max_load_avg, config)
]

manage_ansible = get_default(

Loading…
Cancel
Save