From e2fc03322655ba50646c84f986c3b7846eb69618 Mon Sep 17 00:00:00 2001 From: Kaiyan Sheng Date: Tue, 19 Jan 2016 13:34:28 -0700 Subject: [PATCH] Add Plugin to Monitor File Size This agent plugin is used for gathering the size of individual files or the size of each file under a specific directory. Change-Id: I48174dd16280adc3ef2a2259e740a2708860dd73 --- conf.d/file_size.yaml.example | 45 ++++++++++ monasca_agent/collector/checks_d/file_size.py | 86 +++++++++++++++++++ monasca_setup/detection/plugins/file_size.py | 23 +++++ monasca_setup/detection/service_plugin.py | 52 ++++++++--- monasca_setup/detection/utils.py | 17 +++- 5 files changed, 207 insertions(+), 16 deletions(-) create mode 100644 conf.d/file_size.yaml.example create mode 100644 monasca_agent/collector/checks_d/file_size.py create mode 100644 monasca_setup/detection/plugins/file_size.py diff --git a/conf.d/file_size.yaml.example b/conf.d/file_size.yaml.example new file mode 100644 index 00000000..b960a5fb --- /dev/null +++ b/conf.d/file_size.yaml.example @@ -0,0 +1,45 @@ +# (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP + +init_config: + +instances: + # This config is for the File Check which is used to report metrics + # for the size of the files in a given directory + # + # NOTE: This check is NOT currently supported on Windows systems + # + # For each instance, both parameters 'directory_name' and 'file_names' are + # required + # + # WARNING: Ensure the user account running the Agent (typically mon-agent) + # has read access to the monitored directory and files. + # + # Instances take the following parameters: + # "directory_name" - string, the directory of the files to monitor. + # Required + # "file_names" - list of strings, names of the files to monitor. Required + # "recursive" - boolean, when true and file_name is set to '*' will + # recursively grab files under the given directory to gather + # stats on. + + # Check the size of all the files under directory_1 +# - built_by: Logging +# directory_name: /path/to/directory_1 +# file_names: +# - '*' +# recursive: True + + # Check one file under directory_2 +# - built_by: Logging +# directory_name: /path/to/directory_2 +# file_names: +# -file_name2 +# recursive: False + + # Check two or more files under directory_3 +# - built_by: Logging +# directory_name: /path/to/directory_3 +# file_names: +# -file_name31 +# -file_name32 +# recursive: False \ No newline at end of file diff --git a/monasca_agent/collector/checks_d/file_size.py b/monasca_agent/collector/checks_d/file_size.py new file mode 100644 index 00000000..6d5cb71e --- /dev/null +++ b/monasca_agent/collector/checks_d/file_size.py @@ -0,0 +1,86 @@ +# (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP + +from os.path import abspath +from os.path import exists +from os.path import isfile +from os.path import join + +from os import listdir +from os import stat +from os import walk + +import logging +import monasca_agent.collector.checks as checks + +log = logging.getLogger(__name__) + + +class FileSize(checks.AgentCheck): + + """This check is for monitoring and reporting metrics on the size of + files for a provided directory + + Config options: + "directory_name" - string, directory of the file (required) + "file_names" - list of strings, file names under directory_name to + gather stats for (required) + "recursive" - boolean, when true and file_name is set to '*' will + recursively grab files under the given directory to + gather stats on. + """ + def __init__(self, name, init_config, agent_config, instances=None): + super(FileSize, self).__init__(name, init_config, agent_config, + instances) + + def check(self, instance): + if "directory_name" not in instance: + raise Exception('FileSize Check: missing "directory_name" in ' + 'config') + if "file_names" not in instance: + raise Exception('FileSize Check: missing "file_names" in config') + recursive = instance.get("recursive") or False + directory_name = instance["directory_name"] + abs_directory = abspath(directory_name) + if exists(abs_directory): + if instance["file_names"] != ['*']: + file_names = instance["file_names"] + self._get_stats(abs_directory, file_names, instance) + else: + if recursive: + for root, dirs, files in walk(abs_directory): + self._get_stats(root, files, instance) + else: + files = [file_name for file_name in listdir(abs_directory) + if isfile(join(abs_directory, file_name))] + self._get_stats(abs_directory, files, instance) + else: + log.error('FileSize Check: directory {0} does not exist'. + format(abs_directory)) + + def _get_stats(self, directory_name, files, instance): + num_files = 0 + for file_name in files: + abs_dir_name = abspath(join(directory_name, file_name)) + if isfile(abs_dir_name): + dimensions = self._set_dimensions({ + "file_name": file_name, + "directory_name": directory_name}, instance) + got_stats = False + file_abs_path = join(directory_name, file_name) + try: + file_stat = stat(file_abs_path) + except OSError as ose: + log.warn("FileSize Check: could not stat file %s - %s" % ( + file_abs_path, ose)) + else: + file_bytes = file_stat.st_size + self.gauge("file.size_bytes", file_bytes, + dimensions=dimensions) + got_stats = True + if got_stats: + num_files += 1 + else: + log.error('FileSize Check: file {0} does not exist'. + format(abs_dir_name)) + log.debug('Collected {0} file_size metrics from {1}'. + format(num_files, directory_name)) diff --git a/monasca_setup/detection/plugins/file_size.py b/monasca_setup/detection/plugins/file_size.py new file mode 100644 index 00000000..dc8ea8d3 --- /dev/null +++ b/monasca_setup/detection/plugins/file_size.py @@ -0,0 +1,23 @@ +# (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP + +import monasca_setup.detection + + +class FileSize(monasca_setup.detection.ServicePlugin): + + """Detect FileSize of daemons and setup configuration to monitor them. + file_dirs_names example: + 'file_dirs_names': [('/path/to/directory_1', ['*'], True), + ('/path/to/directory_2', ['file_name2'], False), + ('/path/to/directory_3', ['file_name31', 'file_name32'])] + """ + + def __init__(self, template_dir, overwrite=True, args=None): + service_params = { + 'args': args, + 'template_dir': template_dir, + 'overwrite': overwrite, + 'service_name': 'file-size-service', + 'file_dirs_names': [], + 'search_pattern': ''} + super(FileSize, self).__init__(service_params) diff --git a/monasca_setup/detection/service_plugin.py b/monasca_setup/detection/service_plugin.py index d910bff7..4fe7d59d 100644 --- a/monasca_setup/detection/service_plugin.py +++ b/monasca_setup/detection/service_plugin.py @@ -1,4 +1,4 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +# (C) Copyright 2015-2016 Hewlett Packard Enterprise Development Company LP import logging import psutil @@ -9,9 +9,9 @@ from plugin import Plugin from monasca_setup import agent_config from monasca_setup.detection.utils import find_process_cmdline from monasca_setup.detection.utils import service_api_check +from monasca_setup.detection.utils import watch_file_size from monasca_setup.detection.utils import watch_process - log = logging.getLogger(__name__) @@ -25,7 +25,8 @@ class ServicePlugin(Plugin): def __init__(self, kwargs): self.service_name = kwargs['service_name'] - self.process_names = kwargs['process_names'] + self.process_names = kwargs.get('process_names') + self.file_dirs_names = kwargs.get('file_dirs_names') self.service_api_url = kwargs.get('service_api_url') self.search_pattern = kwargs['search_pattern'] overwrite = kwargs['overwrite'] @@ -36,12 +37,14 @@ class ServicePlugin(Plugin): try: # Turn 'service_api_url=url' into # dict {'service_api_url':'url'} - args_dict = dict([item.split('=') for item - in args.split()]) + args_dict = dict( + [item.split('=') for item in args.split()]) # Allow args to override all of these parameters if 'process_names' in args_dict: self.process_names = args_dict['process_names'].split(',') + if 'file_dirs_names' in args_dict: + self.file_dirs_names = args_dict['file_dirs_names'] if 'service_api_url' in args_dict: self.service_api_url = args_dict['service_api_url'] if 'search_pattern' in args_dict: @@ -60,11 +63,13 @@ class ServicePlugin(Plugin): """ self.found_processes = [] - - for process in self.process_names: - if find_process_cmdline(process) is not None: - self.found_processes.append(process) - if len(self.found_processes) > 0: + if self.process_names: + for process in self.process_names: + if find_process_cmdline(process) is not None: + self.found_processes.append(process) + if len(self.found_processes) > 0: + self.available = True + if self.file_dirs_names: self.available = True def build_config(self): @@ -72,10 +77,29 @@ class ServicePlugin(Plugin): """ config = agent_config.Plugins() - for process in self.found_processes: - # Watch the service processes - log.info("\tMonitoring the {0} {1} process.".format(process, self.service_name)) - config.merge(watch_process([process], self.service_name, process, exact_match=False)) + if self.found_processes: + for process in self.found_processes: + # Watch the service processes + log.info("\tMonitoring the {0} {1} process.".format(process, self.service_name)) + config.merge(watch_process([process], self.service_name, process, exact_match=False)) + + if self.file_dirs_names: + for file_dir_name in self.file_dirs_names: + # Watch file size + file_dir = file_dir_name[0] + file_names = file_dir_name[1] + if len(file_dir_name) == 3: + file_recursive = file_dir_name[2] + else: + file_recursive = False + if file_names == ['*']: + log.info("\tMonitoring the size of all the files in the " + "directory {0}.".format(file_dir)) + else: + log.info("\tMonitoring the size of files {0} in the " + "directory {1}.".format(", ".join(str(name) for name in file_names), file_dir)) + config.merge(watch_file_size(file_dir, file_names, + file_recursive)) # Skip the http_check if disable_http_check is set if self.args is not None and self.args.get('disable_http_check', False): diff --git a/monasca_setup/detection/utils.py b/monasca_setup/detection/utils.py index 3deedf3b..400143f7 100644 --- a/monasca_setup/detection/utils.py +++ b/monasca_setup/detection/utils.py @@ -1,4 +1,4 @@ -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +# (C) Copyright 2015-2016 Hewlett Packard Enterprise Development Company LP """ Util functions to assist in detection. """ @@ -109,6 +109,20 @@ def watch_process_by_username(username, process_name, service=None, component=No return config +def watch_file_size(directory_name, file_names, file_recursive): + """Takes a directory, a list of files, recursive flag and returns a + Plugins object with the config set. + """ + config = agent_config.Plugins() + parameters = {'directory_name': directory_name, + 'file_names': file_names, + 'recursive': file_recursive} + + config['file_size'] = {'init_config': None, + 'instances': [parameters]} + return config + + def service_api_check(name, url, pattern, service=None, component=None): """Setup a service api to be watched by the http_check plugin. """ @@ -125,7 +139,6 @@ def service_api_check(name, url, pattern, service=None, component=None): config['http_check'] = {'init_config': None, 'instances': [parameters]} - return config