Modify the agent and setup to accept custom plugins
For the agent I just modified to the config to use the new standard location for custom checks. For monasca-setup I added automatic plugin detection. Fixed config compare so order changes don't register as a config change. Change-Id: I7ab17c894bb0496e30d7f5aa5a1b1cf9684bdf87
This commit is contained in:
Tim Kuhlman
parent
a3a77ba227
commit
1a73f3425b
|
|
@ -71,9 +71,6 @@ Main:
|
|||
# Change port the Agent is listening to
|
||||
# listen_port: 17123
|
||||
|
||||
# Additional directory to look for checks
|
||||
# additional_checksd: /etc/monasca/agent/checks.d/
|
||||
|
||||
# Allow non-local traffic to this Agent
|
||||
# This is required when using this Agent as a proxy for other Agents
|
||||
# that might not have an internet connection
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
- [AgentCheck Interface](#agentcheck-interface)
|
||||
- [ServicesCheck interface](#servicescheck-interface)
|
||||
- [Submitting Metrics](#submitting-metrics)
|
||||
- [Example Check Plugin](#example-check-plugin)
|
||||
- [Check Plugin Configuration](#check-plugin-configuration)
|
||||
- [init_config](#init_config)
|
||||
- [instances](#instances)
|
||||
|
|
@ -26,6 +27,7 @@
|
|||
- [Plugins Object](#plugins-object)
|
||||
- [Plugin Interface](#plugin-interface)
|
||||
- [Plugin Utilities](#plugin-utilities)
|
||||
- [Example Detection Plugin](#example-detection-plugin)
|
||||
|
||||
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
|
||||
|
||||
|
|
@ -162,6 +164,22 @@ As part of the parent class, you're given a logger at self.log. The log handler
|
|||
|
||||
Of course, when writing your plugin you should ensure that your code raises meaningful exceptions when unanticipated errors occur.
|
||||
|
||||
#### Example Check Plugin
|
||||
/usr/lib/monasca/agent/custom_checks.d/example.py
|
||||
|
||||
```
|
||||
import time
|
||||
import monasca_agent.collector.checks as checks
|
||||
|
||||
|
||||
class Example(checks.AgentCheck):
|
||||
|
||||
def check(self, instance):
|
||||
"""Example stats """
|
||||
dimensions = self._set_dimensions(None, instance)
|
||||
self.gauge('example.time', time.time(), dimensions)
|
||||
```
|
||||
|
||||
#### Check Plugin Configuration
|
||||
Each plugin has a corresponding `yaml` configuration file with the same stem name as the plugin script file.
|
||||
|
||||
|
|
@ -173,9 +191,11 @@ init_config:
|
|||
key2: value2
|
||||
|
||||
instances:
|
||||
- username: john_smith
|
||||
- name: john_smith
|
||||
username: john_smith
|
||||
password: 123456
|
||||
- username: jane_smith
|
||||
- name: jane_smith
|
||||
username: jane_smith
|
||||
password: 789012
|
||||
```
|
||||
|
||||
|
|
@ -185,6 +205,8 @@ In the init_config section you can specify an arbitrary number of global name:va
|
|||
##### instances
|
||||
The instances section is a list of instances that this check will be run against. Your actual check() method is run once per instance. The name:value pairs for each instance specify details about the instance that are necessary for the check.
|
||||
|
||||
It is best practice to include a name for each instance as the monasca-setup program uses this to avoid duplicating instances.
|
||||
|
||||
##### Plugin Documentation
|
||||
Your plugin should include an example `yaml` configuration file to be placed in `/etc/monasca/agent/conf.d` which has the name of the plugin YAML file plus the extension '.example', so the example configuration file for the process plugin would be at `/etc/monasca/agent/conf.d/process.yaml.example. This file should include a set of example init_config and instances clauses that demonstrate how the plugin can be configured.
|
||||
|
||||
|
|
@ -206,3 +228,27 @@ All detection plugins inherit either from the Plugin class found in `monasca_set
|
|||
#### Plugin Utilities
|
||||
|
||||
Useful detection plugin utilities can be found in `monasca_setup/detection/utils.py`. Utilities include functions to find local processes by commandline or name, or who's listening on a particular port, or functions to watch processes or service APIs.
|
||||
|
||||
#### Example Detection Plugin
|
||||
/usr/lib/monasca/agent/custom_detect.d/example.py
|
||||
```
|
||||
import monasca_setup.agent_config
|
||||
import monasca_setup.detection
|
||||
|
||||
|
||||
class Example(monasca_setup.detection.Plugin):
|
||||
"""Configures example check plugin."""
|
||||
def _detect(self):
|
||||
"""Run detection, set self.available True if the service is detected."""
|
||||
self.available = True
|
||||
|
||||
def build_config(self):
|
||||
"""Build the config as a Plugins object and return. """
|
||||
config = monasca_setup.agent_config.Plugins()
|
||||
config['example'] = {'init_config': None,
|
||||
'instances': [{'dimensions':{'example_key':'example_value'}}]}
|
||||
return config
|
||||
|
||||
def dependencies_installed(self):
|
||||
return True
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
import logging
|
||||
import os
|
||||
import pkg_resources
|
||||
import re
|
||||
import six
|
||||
import string
|
||||
import cStringIO as cstringio
|
||||
import yaml
|
||||
|
||||
try:
|
||||
|
|
@ -44,7 +41,7 @@ class Config(object):
|
|||
'dimensions': None,
|
||||
'listen_port': None,
|
||||
'version': self.get_version(),
|
||||
'additional_checksd': os.path.join(os.path.dirname(self._configFile), '/checks_d/'),
|
||||
'additional_checksd': '/usr/lib/monasca/agent/custom_checks.d',
|
||||
'limit_memory_consumption': None,
|
||||
'skip_ssl_validation': False,
|
||||
'watchdog': True,
|
||||
|
|
|
|||
|
|
@ -1,39 +1 @@
|
|||
# Enabled plugins
|
||||
|
||||
from apache import Apache
|
||||
from ceilometer import Ceilometer
|
||||
from cinder import Cinder
|
||||
from glance import Glance
|
||||
from kafka_consumer import Kafka
|
||||
from keystone import Keystone
|
||||
from libvirt import Libvirt
|
||||
from mon import MonAPI, MonPersister, MonThresh
|
||||
from mysql import MySQL
|
||||
from neutron import Neutron
|
||||
from nova import Nova
|
||||
from ntp import Ntp
|
||||
from postfix import Postfix
|
||||
from rabbitmq import RabbitMQ
|
||||
from swift import Swift
|
||||
from system import System
|
||||
from zookeeper import Zookeeper
|
||||
|
||||
DETECTION_PLUGINS = [Apache,
|
||||
Ceilometer,
|
||||
Cinder,
|
||||
Glance,
|
||||
Kafka,
|
||||
Keystone,
|
||||
Libvirt,
|
||||
MonAPI,
|
||||
MonPersister,
|
||||
MonThresh,
|
||||
MySQL,
|
||||
Neutron,
|
||||
Nova,
|
||||
Ntp,
|
||||
Postfix,
|
||||
RabbitMQ,
|
||||
Swift,
|
||||
System,
|
||||
Zookeeper]
|
||||
|
|
|
|||
|
|
@ -1,11 +1,20 @@
|
|||
""" Util functions to assist in detection.
|
||||
"""
|
||||
import glob
|
||||
import imp
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import psutil
|
||||
import subprocess
|
||||
|
||||
from monasca_setup import agent_config
|
||||
from plugin import Plugin
|
||||
from subprocess import Popen, PIPE, CalledProcessError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# check_output was introduced in python 2.7, function added
|
||||
# to accommodate python 2.6
|
||||
try:
|
||||
|
|
@ -25,6 +34,38 @@ except AttributeError:
|
|||
return output
|
||||
|
||||
|
||||
def find_plugins(custom_path):
|
||||
""" Find and import all detection plugins. It will look in detection/plugins dir of the code as well as custom_path
|
||||
|
||||
:param custom_path: An additional path to search for detection plugins
|
||||
:return: A list of imported detection plugin classes.
|
||||
"""
|
||||
|
||||
# This was adapted from what monasca_agent.common.util.load_check_directory
|
||||
plugin_paths = glob.glob(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'plugins', '*.py'))
|
||||
plugin_paths.extend(glob.glob(os.path.join(custom_path, '*.py')))
|
||||
|
||||
plugins = []
|
||||
|
||||
for plugin_path in plugin_paths:
|
||||
if os.path.basename(plugin_path) == '__init__.py':
|
||||
continue
|
||||
try:
|
||||
plugin = imp.load_source(os.path.splitext(os.path.basename(plugin_path))[0], plugin_path)
|
||||
except Exception:
|
||||
log.exception('Unable to import detection plugin {0}'.format(plugin_path))
|
||||
|
||||
# Verify this is a subclass of Plugin
|
||||
classes = inspect.getmembers(plugin, inspect.isclass)
|
||||
for _, clsmember in classes:
|
||||
if Plugin == clsmember:
|
||||
continue
|
||||
if issubclass(clsmember, Plugin):
|
||||
plugins.append(clsmember)
|
||||
|
||||
return plugins
|
||||
|
||||
|
||||
def find_process_cmdline(search_string):
|
||||
"""Simple function to search running process for one with cmdline containing.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -12,12 +12,13 @@ import sys
|
|||
import yaml
|
||||
|
||||
import agent_config
|
||||
from detection.plugins import DETECTION_PLUGINS
|
||||
from detection.utils import find_plugins
|
||||
from service.detection import detect_init
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CUSTOM_PLUGIN_PATH = '/usr/lib/monasca/agent/custom_detect.d'
|
||||
# dirname is called twice to get the dir 1 above the location of the script
|
||||
PREFIX_DIR = os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0])))
|
||||
|
||||
|
|
@ -137,22 +138,23 @@ def main(argv=None):
|
|||
|
||||
# Run through detection and config building for the plugins
|
||||
plugin_config = agent_config.Plugins()
|
||||
detected_plugins = find_plugins(CUSTOM_PLUGIN_PATH)
|
||||
if args.system_only:
|
||||
from detection.plugins.system import System
|
||||
plugins = [System]
|
||||
elif args.detection_plugins is not None:
|
||||
lower_plugins = [p.lower() for p in args.detection_plugins]
|
||||
plugins = []
|
||||
for plugin in DETECTION_PLUGINS:
|
||||
for plugin in detected_plugins:
|
||||
if plugin.__name__.lower() in lower_plugins:
|
||||
plugins.append(plugin)
|
||||
|
||||
if len(plugins) != len(args.detection_plugins):
|
||||
plugin_names = [p.__name__ for p in DETECTION_PLUGINS]
|
||||
plugin_names = [p.__name__ for p in detected_plugins]
|
||||
log.warn("Not all plugins found, discovered plugins {0}\nAvailable plugins{1}".format(plugins,
|
||||
plugin_names))
|
||||
else:
|
||||
plugins = DETECTION_PLUGINS
|
||||
plugins = detected_plugins
|
||||
|
||||
for detect_class in plugins:
|
||||
detect = detect_class(args.template_dir, args.overwrite)
|
||||
|
|
@ -174,6 +176,12 @@ def main(argv=None):
|
|||
old_config = yaml.load(config_file.read())
|
||||
if old_config is not None:
|
||||
agent_config.merge_by_name(value['instances'], old_config['instances'])
|
||||
# Sort before compare, if instances have no name the sort will fail making order changes significant
|
||||
try:
|
||||
value['instances'].sort(key=lambda k: k['name'])
|
||||
old_config['instances'].sort(key=lambda k: k['name'])
|
||||
except Exception:
|
||||
pass
|
||||
if value == old_config: # Don't write config if no change
|
||||
continue
|
||||
with open(config_path, 'w') as config_file:
|
||||
|
|
|
|||
Loading…
Reference in New Issue