From a2a2a88887c6f2761d6f6ec1f339cf949207c0b2 Mon Sep 17 00:00:00 2001 From: Eric MacDonald Date: Tue, 26 Jan 2021 07:58:05 -0500 Subject: [PATCH] Avoid loading collectd's default plugins The current opensource collectd rpm installs several default plugins, some that overlap starlingx developed plugins and others that simply collect way too much data. The plugins in question are: /etc/collectd.d/90-default-plugins-syslog.conf /etc/collectd.d/90-default-plugins-memory.conf /etc/collectd.d/90-default-plugins-load.conf /etc/collectd.d/90-default-plugins-interface.conf /etc/collectd.d/90-default-plugins-cpu.conf This update moves the value added starlingx plugins to /etc/collectd.d/starlingx and relies another puppet update to change the collectd's plugin search path accordingly. Test Plan: PASS: Verify default plugins are not loaded and they samples are not collected. PASS: Verify patch apply and remove. Note: is reboot required patch PASS: Verify the daily influxdb usage drops by 80-85%. Regression: PASS: Verify collectd alarm/degrade regression soak Change-Id: Ic7884ae69014fa274f0bd0515adec90b08747c67 Closes-Bug: 1905581 Signed-off-by: Eric MacDonald --- collectd-extensions/centos/build_srpm.data | 1 + .../centos/collectd-extensions.spec | 42 +++++++++---------- collectd-extensions/src/README | 6 +++ collectd-extensions/src/fm_notifier.py | 19 ++++++--- 4 files changed, 41 insertions(+), 27 deletions(-) create mode 100644 collectd-extensions/src/README diff --git a/collectd-extensions/centos/build_srpm.data b/collectd-extensions/centos/build_srpm.data index 2fce999..11618ac 100644 --- a/collectd-extensions/centos/build_srpm.data +++ b/collectd-extensions/centos/build_srpm.data @@ -1,6 +1,7 @@ SRC_DIR="$PKG_BASE" COPY_LIST="$PKG_BASE/src/LICENSE \ + $PKG_BASE/src/README \ $PKG_BASE/src/collectd.conf.pmon \ $PKG_BASE/src/collectd.service \ $PKG_BASE/src/fm_notifier.py \ diff --git a/collectd-extensions/centos/collectd-extensions.spec b/collectd-extensions/centos/collectd-extensions.spec index cbbbc45..635a0c9 100644 --- a/collectd-extensions/centos/collectd-extensions.spec +++ b/collectd-extensions/centos/collectd-extensions.spec @@ -15,6 +15,7 @@ Source2: collectd.conf.pmon # collectd python plugin files - notifiers Source3: fm_notifier.py Source5: plugin_common.py +Source6: README # collectd python plugin files - resource plugins Source11: cpu.py @@ -53,7 +54,8 @@ StarlingX collectd extensions %define debug_package %{nil} %define local_unit_dir %{_sysconfdir}/systemd/system -%define local_plugin_dir %{_sysconfdir}/collectd.d +%define local_default_plugin_dir %{_sysconfdir}/collectd.d +%define local_starlingx_plugin_dir %{_sysconfdir}/collectd.d/starlingx %define local_python_extensions_dir /opt/collectd/extensions/python %define local_config_extensions_dir /opt/collectd/extensions/config @@ -65,7 +67,8 @@ StarlingX collectd extensions %install install -m 755 -d %{buildroot}%{_sysconfdir} install -m 755 -d %{buildroot}%{local_unit_dir} -install -m 755 -d %{buildroot}%{local_plugin_dir} +install -m 755 -d %{buildroot}%{local_default_plugin_dir} +install -m 755 -d %{buildroot}%{local_starlingx_plugin_dir} install -m 755 -d %{buildroot}%{local_config_extensions_dir} install -m 755 -d %{buildroot}%{local_python_extensions_dir} @@ -77,6 +80,9 @@ install -m 600 %{SOURCE2} %{buildroot}%{local_config_extensions_dir} install -m 700 %{SOURCE3} %{buildroot}%{local_python_extensions_dir} install -m 700 %{SOURCE5} %{buildroot}%{local_python_extensions_dir} +# install README file into /etc/collectd.d +install -m 644 %{SOURCE6} %{buildroot}%{local_default_plugin_dir} + # collectd python plugin files - resource plugins install -m 700 %{SOURCE11} %{buildroot}%{local_python_extensions_dir} install -m 700 %{SOURCE12} %{buildroot}%{local_python_extensions_dir} @@ -87,24 +93,16 @@ install -m 700 %{SOURCE18} %{buildroot}%{local_python_extensions_dir} install -m 700 %{SOURCE19} %{buildroot}%{local_python_extensions_dir} -# collectd plugin conf files into /etc/collectd.d -install -m 600 %{SOURCE100} %{buildroot}%{local_plugin_dir} -install -m 600 %{SOURCE101} %{buildroot}%{local_plugin_dir} -install -m 600 %{SOURCE102} %{buildroot}%{local_plugin_dir} -install -m 600 %{SOURCE103} %{buildroot}%{local_plugin_dir} -install -m 600 %{SOURCE105} %{buildroot}%{local_plugin_dir} -install -m 600 %{SOURCE106} %{buildroot}%{local_plugin_dir} -install -m 600 %{SOURCE107} %{buildroot}%{local_plugin_dir} -install -m 600 %{SOURCE108} %{buildroot}%{local_plugin_dir} -install -m 600 %{SOURCE109} %{buildroot}%{local_plugin_dir} - -%pre -rm -f /etc/collectd.d/90-default-plugins-syslog.conf -rm -f /etc/collectd.d/90-default-plugins-memory.conf -rm -f /etc/collectd.d/90-default-plugins-load.conf -rm -f /etc/collectd.d/90-default-plugins-interface.conf -rm -f /etc/collectd.d/90-default-plugins-cpu.conf - +# collectd plugin conf files into /etc/collectd.d/starlingx +install -m 600 %{SOURCE100} %{buildroot}%{local_starlingx_plugin_dir} +install -m 600 %{SOURCE101} %{buildroot}%{local_starlingx_plugin_dir} +install -m 600 %{SOURCE102} %{buildroot}%{local_starlingx_plugin_dir} +install -m 600 %{SOURCE103} %{buildroot}%{local_starlingx_plugin_dir} +install -m 600 %{SOURCE105} %{buildroot}%{local_starlingx_plugin_dir} +install -m 600 %{SOURCE106} %{buildroot}%{local_starlingx_plugin_dir} +install -m 600 %{SOURCE107} %{buildroot}%{local_starlingx_plugin_dir} +install -m 600 %{SOURCE108} %{buildroot}%{local_starlingx_plugin_dir} +install -m 600 %{SOURCE109} %{buildroot}%{local_starlingx_plugin_dir} %clean rm -rf $RPM_BUILD_ROOT @@ -112,6 +110,8 @@ rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root,-) %config(noreplace) %{local_unit_dir}/collectd.service -%{local_plugin_dir}/* +%{local_default_plugin_dir}/* +%dir %{local_starlingx_plugin_dir} +%{local_starlingx_plugin_dir}/* %{local_config_extensions_dir}/* %{local_python_extensions_dir}/* diff --git a/collectd-extensions/src/README b/collectd-extensions/src/README new file mode 100644 index 0000000..79c5c36 --- /dev/null +++ b/collectd-extensions/src/README @@ -0,0 +1,6 @@ +The upstream default plugin loader files in this +directory are not used. + +The /etc/collectd.conf file is updated to load its +plugins from /etc/collectd.d/starlingx while the +starlingx collectd-extensions package is installed. diff --git a/collectd-extensions/src/fm_notifier.py b/collectd-extensions/src/fm_notifier.py index 4e06d89..8948c24 100755 --- a/collectd-extensions/src/fm_notifier.py +++ b/collectd-extensions/src/fm_notifier.py @@ -122,9 +122,6 @@ PLUGIN = 'alarm notifier' # This plugin's degrade function PLUGIN_DEGRADE = 'degrade notifier' -# Path to the plugin's drop dir -PLUGIN_PATH = '/etc/collectd.d/' - # the name of the collectd samples database DATABASE_NAME = 'collectd samples' @@ -550,7 +547,7 @@ class fmAlarmObject: lock = None # global lock for mread_func mutex database_setup = False # state of database setup database_setup_in_progress = False # connection mutex - + plugin_path = None fm_connectivity = False def __init__(self, id, plugin): @@ -1239,7 +1236,7 @@ class fmAlarmObject: if self.id == ALARM_ID__DF: # read the df.conf file and return/get a list of mount points - conf_file = PLUGIN_PATH + 'df.conf' + conf_file = fmAlarmObject.plugin_path + 'df.conf' if not os.path.exists(conf_file): collectd.error("%s cannot create filesystem " "instance objects ; missing : %s" % @@ -1424,7 +1421,7 @@ def _build_entity_id(plugin, plugin_instance): def _get_df_mountpoints(): - conf_file = PLUGIN_PATH + 'df.conf' + conf_file = fmAlarmObject.plugin_path + 'df.conf' if not os.path.exists(conf_file): collectd.error("%s cannot create filesystem " "instance objects ; missing : %s" % @@ -1645,6 +1642,16 @@ def init_func(): collectd.info("%s %s:%s init function" % (PLUGIN, tsc.nodetype, fmAlarmObject.host)) + # The path to where collectd is looking for its plugins is specified + # at the end of the /etc/collectd.conf file. + # Because so we search for the 'Include' label in reverse order. + for line in reversed(open("/etc/collectd.conf", 'r').readlines()): + if line.startswith('Include'): + plugin_path = line.split(' ')[1].strip("\n").strip('"') + '/' + fmAlarmObject.plugin_path = plugin_path + collectd.info("plugin path: %s" % fmAlarmObject.plugin_path) + break + # Constant CPU Plugin Object Settings obj = PLUGINS[PLUGIN__CPU] obj.resource_name = "Platform CPU"