Merge "Overriding max. starting builds."
This commit is contained in:
commit
6551f35c35
|
@ -688,6 +688,20 @@ The following sections of ``zuul.conf`` are used by the executor:
|
||||||
The executor will observe system load and determine whether
|
The executor will observe system load and determine whether
|
||||||
to accept more jobs every 30 seconds.
|
to accept more jobs every 30 seconds.
|
||||||
|
|
||||||
|
.. attr:: max_starting_builds
|
||||||
|
:default: None
|
||||||
|
|
||||||
|
An executor is accepting up to as many starting builds as defined by the
|
||||||
|
:attr:`executor.load_multiplier` on systems with more than four CPU cores,
|
||||||
|
and up to twice as many on systems with four or less CPU cores. For
|
||||||
|
example, on a system with two CPUs: 2 * 2.5 * 2 - up to ten starting
|
||||||
|
builds may run on such executor; on systems with eight CPUs: 2.5 * 8 - up
|
||||||
|
to twenty starting builds may run on such executor.
|
||||||
|
|
||||||
|
On systems with high CPU/vCPU count an executor may accept too many
|
||||||
|
starting builds. This can be overwritten using this option providing a
|
||||||
|
fixed number of maximum starting builds on an executor.
|
||||||
|
|
||||||
.. attr:: min_avail_hdd
|
.. attr:: min_avail_hdd
|
||||||
:default: 5.0
|
:default: 5.0
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
The maximum starting builds depending on cpu cores can be limited using
|
||||||
|
`executor.max_starting_builds` configuration.
|
|
@ -462,6 +462,58 @@ class TestExecutorHostname(ZuulTestCase):
|
||||||
self.executor_server.hostname)
|
self.executor_server.hostname)
|
||||||
|
|
||||||
|
|
||||||
|
class TestStartingBuildsSensor(ZuulTestCase):
|
||||||
|
config_file = 'zuul.conf'
|
||||||
|
tenant_config_file = 'config/governor/main.yaml'
|
||||||
|
|
||||||
|
def test_default_case(self):
|
||||||
|
# Given
|
||||||
|
cores = multiprocessing.cpu_count()
|
||||||
|
|
||||||
|
# When
|
||||||
|
sensor = StartingBuildsSensor(None, cores * 2.5, None)
|
||||||
|
|
||||||
|
# Then
|
||||||
|
self.assertEqual(sensor.max_starting_builds, int(cores * 2.5 * 2))
|
||||||
|
self.assertEqual(sensor.min_starting_builds, max(int(cores / 2), 1))
|
||||||
|
|
||||||
|
def test_configuration_not_exists(self):
|
||||||
|
# Given
|
||||||
|
cores = multiprocessing.cpu_count()
|
||||||
|
|
||||||
|
# When
|
||||||
|
sensor = StartingBuildsSensor(None, cores * 2.5, self.config)
|
||||||
|
|
||||||
|
# Then
|
||||||
|
self.assertEqual(sensor.max_starting_builds, int(cores * 2.5 * 2))
|
||||||
|
self.assertEqual(sensor.min_starting_builds, max(int(cores / 2), 1))
|
||||||
|
|
||||||
|
def test_configuration_override(self):
|
||||||
|
# Given
|
||||||
|
cores = multiprocessing.cpu_count()
|
||||||
|
self.config.set('executor', 'max_starting_builds', '5')
|
||||||
|
|
||||||
|
# When
|
||||||
|
sensor = StartingBuildsSensor(None, cores * 2.5, self.config)
|
||||||
|
|
||||||
|
# Then
|
||||||
|
self.assertEqual(sensor.max_starting_builds, 5)
|
||||||
|
self.assertEqual(sensor.min_starting_builds, min(
|
||||||
|
max(int(cores / 2), 1), sensor.max_starting_builds))
|
||||||
|
|
||||||
|
def test_configuration_override_affecting_min(self):
|
||||||
|
# Given
|
||||||
|
cores = multiprocessing.cpu_count()
|
||||||
|
self.config.set('executor', 'max_starting_builds', '1')
|
||||||
|
|
||||||
|
# When
|
||||||
|
sensor = StartingBuildsSensor(None, cores * 2.5, self.config)
|
||||||
|
|
||||||
|
# Then
|
||||||
|
self.assertEqual(sensor.max_starting_builds, 1)
|
||||||
|
self.assertEqual(sensor.min_starting_builds, 1)
|
||||||
|
|
||||||
|
|
||||||
class TestGovernor(ZuulTestCase):
|
class TestGovernor(ZuulTestCase):
|
||||||
config_file = 'zuul-executor-hostname.conf'
|
config_file = 'zuul-executor-hostname.conf'
|
||||||
tenant_config_file = 'config/governor/main.yaml'
|
tenant_config_file = 'config/governor/main.yaml'
|
||||||
|
@ -576,7 +628,7 @@ class TestGovernor(ZuulTestCase):
|
||||||
self.log.debug("Waiting for %s to start", jobname)
|
self.log.debug("Waiting for %s to start", jobname)
|
||||||
timeout = time.time() + 30
|
timeout = time.time() + 30
|
||||||
build = None
|
build = None
|
||||||
while (time.time() < timeout and not build):
|
while time.time() < timeout and not build:
|
||||||
for b in self.builds:
|
for b in self.builds:
|
||||||
if b.name == jobname:
|
if b.name == jobname:
|
||||||
build = b
|
build = b
|
||||||
|
|
|
@ -16,15 +16,22 @@ import logging
|
||||||
import multiprocessing
|
import multiprocessing
|
||||||
|
|
||||||
from zuul.executor.sensors import SensorInterface
|
from zuul.executor.sensors import SensorInterface
|
||||||
|
from zuul.lib.config import get_default
|
||||||
|
|
||||||
|
|
||||||
class StartingBuildsSensor(SensorInterface):
|
class StartingBuildsSensor(SensorInterface):
|
||||||
log = logging.getLogger("zuul.executor.sensor.startingbuilds")
|
log = logging.getLogger("zuul.executor.sensor.startingbuilds")
|
||||||
|
|
||||||
def __init__(self, executor, max_load_avg):
|
def __init__(self, executor, max_load_avg, config=None):
|
||||||
self.executor = executor
|
self.executor = executor
|
||||||
self.max_starting_builds = max_load_avg * 2
|
|
||||||
self.min_starting_builds = max(int(multiprocessing.cpu_count() / 2), 1)
|
self.max_starting_builds = get_default(
|
||||||
|
config, 'executor', 'max_starting_builds', int(max_load_avg * 2)) \
|
||||||
|
if config is not None else int(max_load_avg * 2)
|
||||||
|
|
||||||
|
self.min_starting_builds = min(
|
||||||
|
max(int(multiprocessing.cpu_count() / 2), 1),
|
||||||
|
self.max_starting_builds)
|
||||||
|
|
||||||
def _getStartingBuilds(self):
|
def _getStartingBuilds(self):
|
||||||
starting_builds = 0
|
starting_builds = 0
|
||||||
|
|
|
@ -2394,7 +2394,7 @@ class ExecutorServer(object):
|
||||||
HDDSensor(config),
|
HDDSensor(config),
|
||||||
self.pause_sensor,
|
self.pause_sensor,
|
||||||
RAMSensor(config),
|
RAMSensor(config),
|
||||||
StartingBuildsSensor(self, cpu_sensor.max_load_avg)
|
StartingBuildsSensor(self, cpu_sensor.max_load_avg, config)
|
||||||
]
|
]
|
||||||
|
|
||||||
manage_ansible = get_default(
|
manage_ansible = get_default(
|
||||||
|
|
Loading…
Reference in New Issue