openstack/monasca-agent
Code Issues Proposed changes

Add Plugin to Monitor File Size

Browse Source 
This agent plugin is used for gathering the size of individual
files or the size of each file under a specific directory.

Change-Id: I48174dd16280adc3ef2a2259e740a2708860dd73
This commit is contained in:
Kaiyan Sheng 2016-01-19 13:34:28 -07:00
parent b2211aa0eb
commit e2fc033226
5 changed files with 207 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
conf.d/file_size.yaml.example Normal file
View File

@ -0,0 +1,45 @@
# (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP
init_config:
instances:
# This config is for the File Check which is used to report metrics
# for the size of the files in a given directory
#
# NOTE: This check is NOT currently supported on Windows systems
#
# For each instance, both parameters 'directory_name' and 'file_names' are
# required
#
# WARNING: Ensure the user account running the Agent (typically mon-agent)
# has read access to the monitored directory and files.
#
# Instances take the following parameters:
# "directory_name" - string, the directory of the files to monitor.
# Required
# "file_names" - list of strings, names of the files to monitor. Required
# "recursive" - boolean, when true and file_name is set to '*' will
# recursively grab files under the given directory to gather
# stats on.
# Check the size of all the files under directory_1
# - built_by: Logging
# directory_name: /path/to/directory_1
# file_names:
# - '*'
# recursive: True
# Check one file under directory_2
# - built_by: Logging
# directory_name: /path/to/directory_2
# file_names:
# -file_name2
# recursive: False
# Check two or more files under directory_3
# - built_by: Logging
# directory_name: /path/to/directory_3
# file_names:
# -file_name31
# -file_name32
# recursive: False

86
monasca_agent/collector/checks_d/file_size.py Normal file
View File

@ -0,0 +1,86 @@
# (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP
from os.path import abspath
from os.path import exists
from os.path import isfile
from os.path import join
from os import listdir
from os import stat
from os import walk
import logging
import monasca_agent.collector.checks as checks
log = logging.getLogger(__name__)
class FileSize(checks.AgentCheck):
"""This check is for monitoring and reporting metrics on the size of
files for a provided directory
Config options:
"directory_name" - string, directory of the file (required)
"file_names" - list of strings, file names under directory_name to
gather stats for (required)
"recursive" - boolean, when true and file_name is set to '*' will
recursively grab files under the given directory to
gather stats on.
"""
def __init__(self, name, init_config, agent_config, instances=None):
super(FileSize, self).__init__(name, init_config, agent_config,
instances)
def check(self, instance):
if "directory_name" not in instance:
raise Exception('FileSize Check: missing "directory_name" in '
'config')
if "file_names" not in instance:
raise Exception('FileSize Check: missing "file_names" in config')
recursive = instance.get("recursive") or False
directory_name = instance["directory_name"]
abs_directory = abspath(directory_name)
if exists(abs_directory):
if instance["file_names"] != ['*']:
file_names = instance["file_names"]
self._get_stats(abs_directory, file_names, instance)
else:
if recursive:
for root, dirs, files in walk(abs_directory):
self._get_stats(root, files, instance)
else:
files = [file_name for file_name in listdir(abs_directory)
if isfile(join(abs_directory, file_name))]
self._get_stats(abs_directory, files, instance)
else:
log.error('FileSize Check: directory {0} does not exist'.
format(abs_directory))
def _get_stats(self, directory_name, files, instance):
num_files = 0
for file_name in files:
abs_dir_name = abspath(join(directory_name, file_name))
if isfile(abs_dir_name):
dimensions = self._set_dimensions({
"file_name": file_name,
"directory_name": directory_name}, instance)
got_stats = False
file_abs_path = join(directory_name, file_name)
try:
file_stat = stat(file_abs_path)
except OSError as ose:
log.warn("FileSize Check: could not stat file %s - %s" % (
file_abs_path, ose))
else:
file_bytes = file_stat.st_size
self.gauge("file.size_bytes", file_bytes,
dimensions=dimensions)
got_stats = True
if got_stats:
num_files += 1
else:
log.error('FileSize Check: file {0} does not exist'.
format(abs_dir_name))
log.debug('Collected {0} file_size metrics from {1}'.
format(num_files, directory_name))

23
monasca_setup/detection/plugins/file_size.py Normal file
View File

@ -0,0 +1,23 @@
# (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP
import monasca_setup.detection
class FileSize(monasca_setup.detection.ServicePlugin):
"""Detect FileSize of daemons and setup configuration to monitor them.
file_dirs_names example:
'file_dirs_names': [('/path/to/directory_1', ['*'], True),
('/path/to/directory_2', ['file_name2'], False),
('/path/to/directory_3', ['file_name31', 'file_name32'])]
"""
def __init__(self, template_dir, overwrite=True, args=None):
service_params = {
'args': args,
'template_dir': template_dir,
'overwrite': overwrite,
'service_name': 'file-size-service',
'file_dirs_names': [],
'search_pattern': ''}
super(FileSize, self).__init__(service_params)

52
monasca_setup/detection/service_plugin.py
View File

@ -1,4 +1,4 @@
# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
# (C) Copyright 2015-2016 Hewlett Packard Enterprise Development Company LP
import logging
import psutil
@ -9,9 +9,9 @@ from plugin import Plugin
from monasca_setup import agent_config
from monasca_setup.detection.utils import find_process_cmdline
from monasca_setup.detection.utils import service_api_check
from monasca_setup.detection.utils import watch_file_size
from monasca_setup.detection.utils import watch_process
log = logging.getLogger(__name__)
@ -25,7 +25,8 @@ class ServicePlugin(Plugin):
def __init__(self, kwargs):
self.service_name = kwargs['service_name']
self.process_names = kwargs['process_names']
self.process_names = kwargs.get('process_names')
self.file_dirs_names = kwargs.get('file_dirs_names')
self.service_api_url = kwargs.get('service_api_url')
self.search_pattern = kwargs['search_pattern']
overwrite = kwargs['overwrite']
@ -36,12 +37,14 @@ class ServicePlugin(Plugin):
try:
# Turn 'service_api_url=url' into
# dict {'service_api_url':'url'}
args_dict = dict([item.split('=') for item
in args.split()])
args_dict = dict(
[item.split('=') for item in args.split()])
# Allow args to override all of these parameters
if 'process_names' in args_dict:
self.process_names = args_dict['process_names'].split(',')
if 'file_dirs_names' in args_dict:
self.file_dirs_names = args_dict['file_dirs_names']
if 'service_api_url' in args_dict:
self.service_api_url = args_dict['service_api_url']
if 'search_pattern' in args_dict:
@ -60,11 +63,13 @@ class ServicePlugin(Plugin):
"""
self.found_processes = []
for process in self.process_names:
if find_process_cmdline(process) is not None:
self.found_processes.append(process)
if len(self.found_processes) > 0:
if self.process_names:
for process in self.process_names:
if find_process_cmdline(process) is not None:
self.found_processes.append(process)
if len(self.found_processes) > 0:
self.available = True
if self.file_dirs_names:
self.available = True
def build_config(self):
@ -72,10 +77,29 @@ class ServicePlugin(Plugin):
"""
config = agent_config.Plugins()
for process in self.found_processes:
# Watch the service processes
log.info("\tMonitoring the {0} {1} process.".format(process, self.service_name))
config.merge(watch_process([process], self.service_name, process, exact_match=False))
if self.found_processes:
for process in self.found_processes:
# Watch the service processes
log.info("\tMonitoring the {0} {1} process.".format(process, self.service_name))
config.merge(watch_process([process], self.service_name, process, exact_match=False))
if self.file_dirs_names:
for file_dir_name in self.file_dirs_names:
# Watch file size
file_dir = file_dir_name[0]
file_names = file_dir_name[1]
if len(file_dir_name) == 3:
file_recursive = file_dir_name[2]
else:
file_recursive = False
if file_names == ['*']:
log.info("\tMonitoring the size of all the files in the "
"directory {0}.".format(file_dir))
else:
log.info("\tMonitoring the size of files {0} in the "
"directory {1}.".format(", ".join(str(name) for name in file_names), file_dir))
config.merge(watch_file_size(file_dir, file_names,
file_recursive))
# Skip the http_check if disable_http_check is set
if self.args is not None and self.args.get('disable_http_check', False):

17
monasca_setup/detection/utils.py
View File

@ -1,4 +1,4 @@
# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
# (C) Copyright 2015-2016 Hewlett Packard Enterprise Development Company LP
""" Util functions to assist in detection.
"""
@ -109,6 +109,20 @@ def watch_process_by_username(username, process_name, service=None, component=No
return config
def watch_file_size(directory_name, file_names, file_recursive):
"""Takes a directory, a list of files, recursive flag and returns a
Plugins object with the config set.
"""
config = agent_config.Plugins()
parameters = {'directory_name': directory_name,
'file_names': file_names,
'recursive': file_recursive}
config['file_size'] = {'init_config': None,
'instances': [parameters]}
return config
def service_api_check(name, url, pattern, service=None, component=None):
"""Setup a service api to be watched by the http_check plugin.
"""
@ -125,7 +139,6 @@ def service_api_check(name, url, pattern, service=None, component=None):
config['http_check'] = {'init_config': None,
'instances': [parameters]}
return config