Add: dynamic import of analyze modules

modules should be put into timmy/analyze_modules directory
subdirectories are also supported

__example__.py explains the internals and can be used as a template

Change-Id: I4d32f2467dfe1885e71340bd24e8e587aed667c1

setup.py

@@ -38,7 +38,10 @@ setup(name=pname,
                    'operations: two-way data transfer, log collection, '
                    'remote command execution'),
       long_description=open('README.md').read(),
-      packages=[pname, '%s.modules' % pname, '%s_data' % pname],
+      packages=[pname,
+                '%s.analyze_modules' % pname,
+                '%s.modules' % pname,
+                '%s_data' % pname],
       install_requires=['pyyaml'],
       include_package_data=True,
       entry_points={'console_scripts': ['%s=%s.cli:main' % (pname, pname)]},

specs/python-timmy.spec

@@ -4,7 +4,7 @@
 %global pypi_name timmy
 
 Name:           python-%{pypi_name}
-Version:        1.25.2
+Version:        1.25.3
 Release:        1%{?dist}~mos0
 Summary:        Log collector tool for OpenStack Fuel
 
@@ -107,6 +107,9 @@ popd
 
 
 %changelog
+* Mon Dec 12 2016 Dmitry Sutyagin <dsutyagin@mirantis.com> - 1.25.3
+- Add: dynamic import of analyze modules
+
 * Thu Dec 9 2016 Aleksandr Dobdin <adobdin@mirantis.com> - 1.25.2
 - Add: fuel network template download script
 

timmy/analyze.py

@@ -15,7 +15,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from timmy.analyze_health import GREEN, UNKNOWN, YELLOW, RED
 from timmy.env import project_name
+import imp
 import logging
 import os
 import sys
@@ -25,70 +27,20 @@ logger = logging.getLogger(project_name)
 
 
 def analyze(node_manager):
-    col_msg = 'Column "%s" not found in output of "%s" from node "%s"'
+    def is_module(f):
-    green = 0
+        return f.endswith('.py') and not f.startswith('__')
-    unknown = 1
-    yellow = 2
-    red = 3
 
-    def parse_df_m(data, script, node):
+    fn_mapping = {}
-        column_use = "Use%"
+    modules_dir = 'analyze_modules'
-        full = 100
+    modules_path = os.path.join(os.path.dirname(__file__), modules_dir)
-        near_full = 80
+    module_paths = m = []
-        health = green
+    for item in os.walk(modules_path):
-        details = []
+        m.extend([os.sep.join([item[0], f]) for f in item[2] if is_module(f)])
-        if column_use not in data[0]:
+    for module_path in module_paths:
-            logger.warning(col_msg % (column_use, script, node.repr))
+        module_name = os.path.basename(module_path)
-            health = unknown
+        module = imp.load_source(module_name, module_path)
-        index = data[0].split().index(column_use)
+        module.register(fn_mapping)
-        prepend_str = ''  # workaround for data which spans 2 lines
-        index_shift = 0
-        for line in data[2:]:
-            if len(line.split()) <= index:
-                prepend_str = line.rstrip()
-                index_shift = len(line.split())
-                continue
-            value = int(line.split()[index - index_shift][:-1])
-            if value >= full:
-                health = red
-                details.append(prepend_str + line)
-            elif value >= near_full:
-                health = yellow if health < yellow else health
-                details.append(prepend_str + line)
-            prepend_str = ''
-            index_shift = 0
-        return health, details
 
-    def parse_df_i(data, script, node):
-        column_use = "IUse%"
-        full = 100
-        near_full = 80
-        health = green
-        details = []
-        if column_use not in data[0]:
-            logger.warning(col_msg % (column_use, script, node.repr))
-            health = unknown
-        index = data[0].split().index(column_use)
-        prepend_str = ''  # workaround for data which spans 2 lines
-        index_shift = 0
-        for line in data[2:]:
-            if len(line.split()) <= index:
-                prepend_str = line.rstrip()
-                index_shift = len(line.split())
-                continue
-            if "%" in line.split()[index - index_shift]:
-                value = int(line.split()[index - index_shift][:-1])
-                if value >= full:
-                    health = red
-                    details.append(prepend_str + line)
-                elif value >= near_full:
-                    health = yellow if health < yellow else health
-                    details.append(prepend_str + line)
-            prepend_str = ''
-        return health, details
-
-    fn_mapping = {"df-m": parse_df_m,
-                  "df-i": parse_df_i}
     results = {}
     for node in node_manager.nodes.values():
         if not node.mapscr:
@@ -112,10 +64,10 @@ def analyze(node_manager):
 
 
 def analyze_print_results(node_manager):
-    code_colors = {3: ["RED", "\033[91m"],
+    code_colors = {GREEN: ["GREEN", "\033[92m"],
-                   2: ["YELLOW", "\033[93m"],
+                   UNKNOWN: ["UNKNOWN", "\033[94m"],
-                   0: ["GREEN", "\033[92m"],
+                   YELLOW: ["YELLOW", "\033[93m"],
-                   1: ["BLUE", "\033[94m"]}
+                   RED: ["RED", "\033[91m"]}
     color_end = "\033[0m"
     print("Nodes health analysis:")
     for node, result in node_manager.analyze_results.items():

timmy/analyze_health.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+GREEN = 0
+UNKNOWN = 1
+YELLOW = 2
+RED = 3

timmy/analyze_modules/__example__.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+'''
+please import and use health constants from analyze_health
+GREEN - no issues
+UNKNOWN - cannot determine / cannot parse output
+YELLOW - condition is bad but not critical / impactful
+RED - critical / impactful condition
+
+
+if you want to write log messages, add the following lines:
+from timmy.env import project_name
+import logging
+
+
+logger = logging.getLogger(project_name)
+'''
+from timmy.analyze_health import GREEN, UNKNOWN, YELLOW, RED
+from timmy.env import project_name
+import logging
+
+
+logger = logging.getLogger(project_name)
+
+
+def register(function_mapping):
+    '''
+    this function is mandatory and it's name must be "register"
+    it should have 1 argument which is a dict
+    it should update the dict with a relation between script names and
+        analyzing functions
+    more than one script can be mapped by a single module
+    see script names in timmy_data/rq/scripts folder
+    '''
+    function_mapping['script-basename'] = parsing_function
+
+
+def parsing_function(data, script, node):
+    '''
+    each analyzing function should have 3 arguments:
+    data - list of strings aquired by reading the output file
+    script - path to the script file
+    node - node object
+
+    return should contain 2 values:
+    health - set to one of the imported constants according to the analysis
+    details - a list of strings - an explanatory message or
+        lines which were indicative of the issue
+    '''
+    health = UNKNOWN
+    line = data[0]  # in this example we only look at the first line
+    details = [line]
+    if line.find('error'):
+        health = RED
+        details.append('This is very bad! Do something NOW!!!')
+    elif line.find('warning'):
+        health = YELLOW
+        details.append('Who cares if it is not RED, right? :)')
+    elif line.find('ok'):
+        health = GREEN
+    return health, details

timmy/analyze_modules/df.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from timmy.analyze_health import GREEN, UNKNOWN, YELLOW, RED
+from timmy.env import project_name
+import logging
+
+
+logger = logging.getLogger(project_name)
+
+col_msg = 'Column "%s" not found in output of "%s" from node "%s"'
+
+
+def register(function_mapping):
+    function_mapping['df-m'] = parse_df_m
+    function_mapping['df-i'] = parse_df_i
+
+
+def parse_df_m(data, script, node):
+    column_use = "Use%"
+    full = 100
+    near_full = 80
+    health = GREEN
+    details = []
+    if column_use not in data[0]:
+        logger.warning(col_msg % (column_use, script, node.repr))
+        health = UNKNOWN
+    index = data[0].split().index(column_use)
+    prepend_str = ''  # workaround for data which spans 2 lines
+    index_shift = 0
+    for line in data[2:]:
+        if len(line.split()) <= index:
+            prepend_str = line.rstrip()
+            index_shift = len(line.split())
+            continue
+        value = int(line.split()[index - index_shift][:-1])
+        if value >= full:
+            health = RED
+            details.append(prepend_str + line)
+        elif value >= near_full:
+            health = YELLOW if health < YELLOW else health
+            details.append(prepend_str + line)
+        prepend_str = ''
+        index_shift = 0
+    return health, details
+
+
+def parse_df_i(data, script, node):
+    column_use = "IUse%"
+    full = 100
+    near_full = 80
+    health = GREEN
+    details = []
+    if column_use not in data[0]:
+        logger.warning(col_msg % (column_use, script, node.repr))
+        health = UNKNOWN
+    index = data[0].split().index(column_use)
+    prepend_str = ''  # workaround for data which spans 2 lines
+    index_shift = 0
+    for line in data[2:]:
+        if len(line.split()) <= index:
+            prepend_str = line.rstrip()
+            index_shift = len(line.split())
+            continue
+        if "%" in line.split()[index - index_shift]:
+            value = int(line.split()[index - index_shift][:-1])
+            if value >= full:
+                health = RED
+                details.append(prepend_str + line)
+            elif value >= near_full:
+                health = YELLOW if health < YELLOW else health
+                details.append(prepend_str + line)
+        prepend_str = ''
+    return health, details

timmy/env.py

@@ -16,7 +16,7 @@
 #    under the License.
 
 project_name = 'timmy'
-version = '1.25.2'
+version = '1.25.3'
 
 if __name__ == '__main__':
     import sys