From 608b22f577d9befe663ebf7f3717fb42ac61f375 Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Mon, 25 Jun 2018 17:11:45 -0400 Subject: [PATCH] Add min_avail_hdd governor for zuul-executor Using the zuul.executor.state_dir setting from zuul.conf, we can create a new governor to track the amount of space a zuul-executor is using. If we go above the min_avail_hdd space (default 5.0%), we'll stop accepting jobs until space has been reclaimed but the executor. Change-Id: Ieb446397135ee5b138829cd2440b8c86abbb7d56 Signed-off-by: Paul Belanger --- doc/source/admin/components.rst | 10 ++++++ tests/unit/test_executor.py | 28 +++++++++++++++++ zuul/executor/sensors/hdd.py | 54 +++++++++++++++++++++++++++++++++ zuul/executor/server.py | 2 ++ 4 files changed, 94 insertions(+) create mode 100644 zuul/executor/sensors/hdd.py diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst index 98915e9259..1170bede3c 100644 --- a/doc/source/admin/components.rst +++ b/doc/source/admin/components.rst @@ -559,6 +559,16 @@ The following sections of ``zuul.conf`` are used by the executor: The executor will observe system load and determine whether to accept more jobs every 30 seconds. + .. attr:: min_avail_hdd + :default: 5.0 + + This is the minimum percentage of HDD storage available for the + :attr:`executor.state_dir` directory. The executor will stop accepting + more than 1 job at a time until more HDD storage is available. The + available HDD percentage is calculated from the total available + disk space divided by the total real storage capacity multiplied by + 100. + .. attr:: min_avail_mem :default: 5.0 diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py index 0998886ffa..9040251a45 100755 --- a/tests/unit/test_executor.py +++ b/tests/unit/test_executor.py @@ -446,6 +446,7 @@ class TestExecutorHostname(ZuulTestCase): class TestGovernor(ZuulTestCase): + config_file = 'zuul-executor-hostname.conf' tenant_config_file = 'config/governor/main.yaml' @mock.patch('os.getloadavg') @@ -464,6 +465,33 @@ class TestGovernor(ZuulTestCase): self.executor_server.manageLoad() self.assertFalse(self.executor_server.accepting_work) + @mock.patch('os.statvfs') + def test_hdd_governor(self, statvfs_mock): + class Dummy(object): + pass + hdd = Dummy() + hdd.f_frsize = 4096 + hdd.f_blocks = 120920708 + hdd.f_bfree = 95716701 + statvfs_mock.return_value = hdd # 20.84% used + + self.executor_server.manageLoad() + self.assertTrue(self.executor_server.accepting_work) + + self.assertReportedStat( + 'zuul.executor.test-executor-hostname_example_com.pct_used_hdd', + value='2084', kind='g') + + hdd.f_bfree = 5716701 + statvfs_mock.return_value = hdd # 95.27% used + + self.executor_server.manageLoad() + self.assertFalse(self.executor_server.accepting_work) + + self.assertReportedStat( + 'zuul.executor.test-executor-hostname_example_com.pct_used_hdd', + value='9527', kind='g') + def test_pause_governor(self): self.executor_server.manageLoad() self.assertTrue(self.executor_server.accepting_work) diff --git a/zuul/executor/sensors/hdd.py b/zuul/executor/sensors/hdd.py new file mode 100644 index 0000000000..c9476e8aea --- /dev/null +++ b/zuul/executor/sensors/hdd.py @@ -0,0 +1,54 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os + +from zuul.executor.sensors import SensorInterface +from zuul.lib.config import get_default + + +def get_avail_hdd_pct(path): + s = os.statvfs(path) + used = float(s.f_blocks - s.f_bfree) + percent = (used / s.f_blocks) * 100 + + return (100.0 - percent) + + +class HDDSensor(SensorInterface): + log = logging.getLogger("zuul.executor.sensor.hdd") + + def __init__(self, config=None): + self.min_avail_hdd = float( + get_default(config, 'executor', 'min_avail_hdd', '5.0')) + self.state_dir = get_default( + config, 'executor', 'state_dir', '/var/lib/zuul', expand_user=True) + + def isOk(self): + avail_hdd_pct = get_avail_hdd_pct(self.state_dir) + + if avail_hdd_pct < self.min_avail_hdd: + return False, "low disk space {:3.1f}% < {}".format( + avail_hdd_pct, self.min_avail_hdd) + + return True, "{:3.1f}% <= {}".format(avail_hdd_pct, self.min_avail_hdd) + + def reportStats(self, statsd, base_key): + avail_hdd_pct = get_avail_hdd_pct(self.state_dir) + + # We multiply the percentage by 100 so we can report it to 2 decimal + # points. + statsd.gauge(base_key + '.pct_used_hdd', + int((100.0 - avail_hdd_pct) * 100)) diff --git a/zuul/executor/server.py b/zuul/executor/server.py index f58537d34d..eb09a5d1ea 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -40,6 +40,7 @@ import gear import zuul.merger.merger import zuul.ansible.logconfig from zuul.executor.sensors.cpu import CPUSensor +from zuul.executor.sensors.hdd import HDDSensor from zuul.executor.sensors.pause import PauseSensor from zuul.executor.sensors.startingbuilds import StartingBuildsSensor from zuul.executor.sensors.ram import RAMSensor @@ -1874,6 +1875,7 @@ class ExecutorServer(object): cpu_sensor = CPUSensor(config) self.sensors = [ cpu_sensor, + HDDSensor(config), self.pause_sensor, RAMSensor(config), StartingBuildsSensor(self, cpu_sensor.max_load_avg)