922a6b53ed
Rather than checking all of the sensors to see if they are okay, then collecting all the data again for stats purposes, do both at the same time. Change-Id: Ia974a7d013057880171fd1695a1d17169d093410
97 lines
3.4 KiB
Python
97 lines
3.4 KiB
Python
# Copyright 2018 BMW Car IT GmbH
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import logging
|
|
import math
|
|
import psutil
|
|
|
|
from zuul.executor.sensors import SensorInterface
|
|
from zuul.lib.config import get_default
|
|
|
|
CGROUP_STATS_FILE = '/sys/fs/cgroup/memory/memory.stat'
|
|
|
|
|
|
def get_avail_mem_pct():
|
|
avail_mem_pct = 100.0 - psutil.virtual_memory().percent
|
|
return avail_mem_pct
|
|
|
|
|
|
class RAMSensor(SensorInterface):
|
|
log = logging.getLogger("zuul.executor.sensor.ram")
|
|
|
|
def __init__(self, statsd, base_key, config=None):
|
|
super().__init__(statsd, base_key)
|
|
self.min_avail_mem = float(get_default(config, 'executor',
|
|
'min_avail_mem', '5.0'))
|
|
self.cgroup_stats_file = CGROUP_STATS_FILE
|
|
|
|
def _read_cgroup_stat(self):
|
|
stat = {}
|
|
try:
|
|
with open(self.cgroup_stats_file) as f:
|
|
for line in f.readlines():
|
|
key, value = line.split(' ')
|
|
stat[key] = int(value.strip())
|
|
except Exception:
|
|
pass
|
|
return stat
|
|
|
|
def _get_cgroup_limit(self):
|
|
stat = self._read_cgroup_stat()
|
|
limit = stat.get('hierarchical_memory_limit', math.inf)
|
|
mem_total = psutil.virtual_memory().total
|
|
if limit < mem_total:
|
|
return limit
|
|
else:
|
|
return math.inf
|
|
|
|
def _get_avail_mem_pct_cgroup(self):
|
|
stat = self._read_cgroup_stat()
|
|
limit = stat.get('hierarchical_memory_limit', math.inf)
|
|
usage = stat.get('total_rss', math.inf)
|
|
|
|
if math.isinf(limit) or math.isinf(usage):
|
|
# pretend we have all memory available if we got infs
|
|
return 100
|
|
|
|
return 100.0 - usage / limit * 100
|
|
|
|
def isOk(self):
|
|
avail_mem_pct = get_avail_mem_pct()
|
|
avail_mem_pct_cgroup = self._get_avail_mem_pct_cgroup()
|
|
|
|
if self.statsd:
|
|
self.statsd.gauge(self.base_key + '.pct_used_ram',
|
|
int((100.0 - avail_mem_pct) * 100))
|
|
if math.isfinite(self._get_cgroup_limit()):
|
|
self.statsd.gauge(self.base_key + '.pct_used_ram_cgroup',
|
|
int((100.0 - avail_mem_pct_cgroup) * 100))
|
|
|
|
if avail_mem_pct < self.min_avail_mem:
|
|
return False, "low memory {:3.1f}% < {}".format(
|
|
avail_mem_pct, self.min_avail_mem)
|
|
|
|
if math.isinf(self._get_cgroup_limit()):
|
|
# we have no cgroup defined limit so we're done now
|
|
return True, "{:3.1f}% <= {}".format(
|
|
avail_mem_pct, self.min_avail_mem)
|
|
|
|
if avail_mem_pct_cgroup < self.min_avail_mem:
|
|
return False, "low memory cgroup {:3.1f}% < {}".format(
|
|
avail_mem_pct_cgroup, self.min_avail_mem)
|
|
|
|
return True, "{:3.1f}% <= {}, {:3.1f}% <= {}".format(
|
|
avail_mem_pct, self.min_avail_mem,
|
|
avail_mem_pct_cgroup, self.min_avail_mem)
|