Add new collectd Dockerfile for rhoso

life of centos stream8 is over. building new dockerfile based on centos stream 9
we will add specific file for ocp worker and baremetal controllers."

Change-Id: Icfeb4e07b8068378e597f546d061e4efc9b2b061
This commit is contained in:
Masco 2024-06-24 12:59:11 +05:30
parent 9327db1522
commit 15e534e67a
2 changed files with 432 additions and 0 deletions

View File

@ -0,0 +1,29 @@
FROM quay.io/centos/centos:stream9
RUN dnf clean all && \
dnf group install -y "Development Tools" --nobest && \
dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
dnf install -y centos-release-opstools && \
dnf install -y collectd collectd-turbostat collectd-disk collectd-apache collectd-ceph \
collectd-mysql collectd-python collectd-ping collectd-virt python3-sqlalchemy-collectd --nobest && \
dnf install -y sysstat && \
dnf install -y python3-pip python3-devel && \
pip3 install --upgrade pip && \
pip3 install pyrabbit && \
dnf install -y perl-DBD-MySQL collectd-dbi && \
dnf install -y centos-release-openstack-bobcat && \
dnf config-manager --set-enabled crb && \
dnf install -y openvswitch libibverbs && \
dnf install -y passwd && \
dnf install -y ceph-common && \
dnf install -y sudo
RUN useradd stack
RUN echo stack | passwd stack --stdin
RUN echo "stack ALL=(root) NOPASSWD:ALL" | tee -a /etc/sudoers.d/stack
RUN chmod 0440 /etc/sudoers.d/stack
RUN rm /etc/collectd.d/virt.conf
ADD files/collectd_iostat_python.py /usr/local/bin/collectd_iostat_python.py
CMD ["collectd", "-f"]

View File

@ -0,0 +1,403 @@
#!/usr/bin/env python
# coding=utf-8
# The MIT License (MIT)
#
# Copyright (c) 2014-2016 Denis Zhdanov
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# collectd-iostat-python
# ======================
#
# Collectd-iostat-python is an iostat plugin for collectd that allows you to
# graph Linux iostat metrics in Graphite or other output formats that are
# supported by collectd.
#
# https://github.com/powdahound/redis-collectd-plugin
# - was used as template
# https://github.com/keirans/collectd-iostat/
# - was used as inspiration and contains some code from
# https://bitbucket.org/jakamkon/python-iostat
# - by Kuba Kończyk <jakamkon at users.sourceforge.net>
#
import signal
import string
import subprocess
import sys
import re
try:
import pyudev
pyudev_available = True
except ImportError:
pyudev_available = False
# Original Version/Author
__version__ = '0.0.5'
__author__ = 'denis.zhdanov@gmail.com'
class IOStatError(Exception):
pass
class CmdError(IOStatError):
pass
class ParseError(IOStatError):
pass
class IOStat(object):
def __init__(self, path='/usr/bin/iostat', interval=2, count=2, disks=[], no_dm_name=False):
self.path = path
self.interval = interval
self.count = count
self.disks = disks
self.no_dm_name = no_dm_name
def parse_diskstats(self, input):
"""
Parse iostat -d and -dx output.If there are more
than one series of statistics, get the last one.
By default parse statistics for all available block devices.
@type input: C{string}
@param input: iostat output
@type disks: list of C{string}s
@param input: lists of block devices that
statistics are taken for.
@return: C{dictionary} contains per block device statistics.
Statistics are in form of C{dictonary}.
Main statistics:
tps Blk_read/s Blk_wrtn/s Blk_read Blk_wrtn
Extended staistics (available with post 2.5 kernels):
rrqm/s wrqm/s r/s w/s rsec/s wsec/s rkB/s wkB/s avgrq-sz \
avgqu-sz await svctm %util
See I{man iostat} for more details.
"""
dstats = {}
dsi = input.rfind('Device')
if dsi == -1:
raise ParseError('Unknown input format: %r' % input)
ds = input[dsi:].splitlines()
hdr = ds.pop(0).split()[1:]
for d in ds:
if d:
d = d.split()
d = [re.sub(r',','.',element) for element in d]
dev = d.pop(0)
if (dev in self.disks) or not self.disks:
dstats[dev] = dict([(k, float(v)) for k, v in zip(hdr, d)])
return dstats
def sum_dstats(self, stats, smetrics):
"""
Compute the summary statistics for chosen metrics.
"""
avg = {}
for disk, metrics in stats.iteritems():
for mname, metric in metrics.iteritems():
if mname not in smetrics:
continue
if mname in avg:
avg[mname] += metric
else:
avg[mname] = metric
return avg
def _run(self, options=None):
"""
Run iostat command.
"""
close_fds = 'posix' in sys.builtin_module_names
args = '%s %s %s %s %s' % (
self.path,
''.join(options),
self.interval,
self.count,
' '.join(self.disks))
return subprocess.Popen(
args,
bufsize=1,
shell=True,
stdout=subprocess.PIPE,
close_fds=close_fds)
@staticmethod
def _get_childs_data(child):
"""
Return child's data when available.
"""
(stdout, stderr) = child.communicate()
ecode = child.poll()
if ecode != 0:
raise CmdError('Command %r returned %d' % (child.cmd, ecode))
return stdout
def get_diskstats(self):
"""
Get all available disks statistics that we can get.
iostat -kNd
tps kB_read/s kB_wrtn/s kB_read kB_wrtn
iostat -kNdx
rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz
avgqu-sz await r_await w_await svctm %util
"""
options=['-','k','N','d']
extdoptions=['-','k','N','d','x']
if self.no_dm_name:
options.remove('N')
extdoptions.remove('N')
dstats = self._run(options)
extdstats = self._run(extdoptions)
dsd = self._get_childs_data(dstats).decode("utf-8")
edd = self._get_childs_data(extdstats).decode("utf-8")
ds = self.parse_diskstats(dsd)
eds = self.parse_diskstats(edd)
for dk, dv in ds.items():
if dk in eds:
ds[dk].update(eds[dk])
return ds
class IOMon(object):
def __init__(self):
self.plugin_name = 'collectd-iostat-python'
self.iostat_path = '/usr/bin/iostat'
self.interval = 60.0
self.iostat_interval = 2
self.iostat_count = 2
self.iostat_disks = []
self.iostat_nice_names = False
self.iostat_disks_regex = ''
self.iostat_udevnameattr = ''
self.skip_multipath = False
self.verbose_logging = False
self.iostat_no_dm_name = False
self.names = {
'tps': {'t': 'transfers_per_second'},
'Blk_read/s': {'t': 'blocks_per_second', 'ti': 'read'},
'kB_read/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 1024},
'MB_read/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 1048576},
'Blk_wrtn/s': {'t': 'blocks_per_second', 'ti': 'write'},
'kB_wrtn/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 1024},
'MB_wrtn/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 1048576},
'Blk_read': {'t': 'blocks', 'ti': 'read'},
'kB_read': {'t': 'bytes', 'ti': 'read', 'm': 1024},
'MB_read': {'t': 'bytes', 'ti': 'read', 'm': 1048576},
'Blk_wrtn': {'t': 'blocks', 'ti': 'write'},
'kB_wrtn': {'t': 'bytes', 'ti': 'write', 'm': 1024},
'MB_wrtn': {'t': 'bytes', 'ti': 'write', 'm': 1048576},
'rrqm/s': {'t': 'requests_merged_per_second', 'ti': 'read'},
'wrqm/s': {'t': 'requests_merged_per_second', 'ti': 'write'},
'r/s': {'t': 'per_second', 'ti': 'read'},
'w/s': {'t': 'per_second', 'ti': 'write'},
'rsec/s': {'t': 'sectors_per_second', 'ti': 'read'},
'rkB/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 1024},
'rMB/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 1048576},
'wsec/s': {'t': 'sectors_per_second', 'ti': 'write'},
'wkB/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 1024},
'wMB/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 1048576},
'avgrq-sz': {'t': 'avg_request_size'},
'avgqu-sz': {'t': 'avg_request_queue'},
'await': {'t': 'avg_wait_time'},
'r_await': {'t': 'avg_wait_time', 'ti': 'read'},
'w_await': {'t': 'avg_wait_time', 'ti': 'write'},
'svctm': {'t': 'avg_service_time'},
'%util': {'t': 'percent', 'ti': 'util'}
}
def log_verbose(self, msg):
if not self.verbose_logging:
return
collectd.info('%s plugin [verbose]: %s' % (self.plugin_name, msg))
def configure_callback(self, conf):
"""
Receive configuration block
"""
for node in conf.children:
val = str(node.values[0])
if node.key == 'Path':
self.iostat_path = val
elif node.key == 'Interval':
self.interval = float(val)
elif node.key == 'IostatInterval':
self.iostat_interval = int(float(val))
elif node.key == 'Count':
self.iostat_count = int(float(val))
elif node.key == 'Disks':
self.iostat_disks = val.split(',')
elif node.key == 'NiceNames':
self.iostat_nice_names = val in ['True', 'true']
elif node.key == 'DisksRegex':
self.iostat_disks_regex = val
elif node.key == 'UdevNameAttr':
self.iostat_udevnameattr = val
elif node.key == 'PluginName':
self.plugin_name = val
elif node.key == 'Verbose':
self.verbose_logging = val in ['True', 'true']
elif node.key == 'SkipPhysicalMultipath':
self.skip_multipath = val in [ 'True', 'true' ]
elif node.key == 'NoDisplayDMName':
self.iostat_no_dm_name = val in [ 'True', 'true' ]
else:
collectd.warning(
'%s plugin: Unknown config key: %s.' % (
self.plugin_name,
node.key))
self.log_verbose(
'Configured with iostat=%s, interval=%s, count=%s, disks=%s, '
'disks_regex=%s udevnameattr=%s skip_multipath=%s no_dm_name=%s' % (
self.iostat_path,
self.iostat_interval,
self.iostat_count,
self.iostat_disks,
self.iostat_disks_regex,
self.iostat_udevnameattr,
self.skip_multipath,
self.iostat_no_dm_name))
collectd.register_read(self.read_callback, self.interval)
def dispatch_value(self, plugin_instance, val_type, type_instance, value):
"""
Dispatch a value to collectd
"""
self.log_verbose(
'Sending value: %s-%s.%s=%s' % (
self.plugin_name,
plugin_instance,
'-'.join([val_type, type_instance]),
value))
val = collectd.Values()
val.plugin = self.plugin_name
val.plugin_instance = plugin_instance
val.type = val_type
if len(type_instance):
val.type_instance = type_instance
val.values = [value, ]
val.meta={'0': True}
val.dispatch()
def read_callback(self):
"""
Collectd read callback
"""
self.log_verbose('Read callback called')
iostat = IOStat(
path=self.iostat_path,
interval=self.iostat_interval,
count=self.iostat_count,
disks=self.iostat_disks,
no_dm_name=self.iostat_no_dm_name)
ds = iostat.get_diskstats()
if not ds:
self.log_verbose('%s plugin: No info received.' % self.plugin_name)
return
if self.iostat_udevnameattr and pyudev_available:
context = pyudev.Context()
for disk in ds:
if not re.match(self.iostat_disks_regex, disk):
continue
if self.iostat_udevnameattr and pyudev_available:
device = pyudev.Device.from_device_file(context, "/dev/" + disk)
if self.skip_multipath:
mp_managed = device.get('DM_MULTIPATH_DEVICE_PATH')
if mp_managed and mp_managed == '1':
self.log_verbose('Skipping physical multipath disk %s' % disk)
continue
if self.iostat_udevnameattr:
persistent_name = device.get(self.iostat_udevnameattr)
if not persistent_name:
self.log_verbose('Unable to determine disk name based on UdevNameAttr: %s' % self.iostat_udevnameattr)
persistent_name = disk
else:
persistent_name = disk
for name in ds[disk]:
if self.iostat_nice_names and name in self.names:
val_type = self.names[name]['t']
if 'ti' in self.names[name]:
type_instance = self.names[name]['ti']
else:
type_instance = ''
value = ds[disk][name]
if 'm' in self.names[name]:
value *= self.names[name]['m']
else:
val_type = 'gauge'
tbl = str.maketrans('/-%', '___')
type_instance = name.translate(tbl)
value = ds[disk][name]
self.dispatch_value(
persistent_name, val_type, type_instance, value)
def restore_sigchld():
"""
Restore SIGCHLD handler for python <= v2.6
It will BREAK exec plugin!!!
See https://github.com/deniszh/collectd-iostat-python/issues/2 for details
"""
if sys.version_info[0] == 2 and sys.version_info[1] <= 6:
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
if __name__ == '__main__':
iostat = IOStat()
ds = iostat.get_diskstats()
for disk in ds:
for metric in ds[disk]:
tbl = str.maketrans('/-%', '___')
metric_name = metric.translate(tbl)
print("%s.%s:%s" % (disk, metric_name, ds[disk][metric]))
sys.exit(0)
else:
import collectd
iomon = IOMon()
# Register callbacks
collectd.register_init(restore_sigchld)
collectd.register_config(iomon.configure_callback)