Initial setup of zookeepr and kafka

conf.d/zk.yaml.example

@@ -1,8 +1,8 @@
 init_config:
 
 instances:
-#   -   host: localhost
-#       port: 2181
-#       timeout: 3
+   -   host: localhost
+       port: 2181
+       timeout: 3
 #       dimensions:
 #            dim1: value1

monsetup/README.md

@@ -8,4 +8,8 @@ configuring it to start up on boot.
 - A good system for specifying active style checks for configuration would be great. Active style checks are those
   which run locally but are checking a remote box.
 - The ability to dynamically add detection plugins could be quite valuable. Also it could aid in the active check config.
-    - With the ability to dynamically add I should also include the ability to dynamically remove.
+    - With the ability to dynamically add I should also include the ability to dynamically remove.
+- A config file with metadata to assist in automatic configuration could be quite valuable, for example user/pass for
+  logging into mysql.
+- The current system does not work well in the case of clusters, for example the kafka plugin will end up collecting
+  the same data on each node of a cluster.

monsetup/detection/__init__.py

@@ -41,12 +41,23 @@ class Plugin(object):
             return self.__class__.__name__
 
 
+def find_process_cmdline(search_string):
+    """Simple function to search running process for one with cmdline containing
+    """
+    for process in psutil.process_iter():
+        for arg in process.cmdline():
+            if arg.find(search_string) != -1:
+                return process
+
+    return None
+
+
 def find_process_name(pname):
     """Simple function to search running process for one with pname.
     """
     for process in psutil.process_iter():
         if pname == process.name():
-            return process.pid
+            return process
 
     return None
 

monsetup/detection/kafka.py

@@ -0,0 +1,66 @@
+import collections
+import logging
+
+from . import Plugin, find_process_cmdline, watch_process
+from monsetup import agent_config
+
+log = logging.getLogger(__name__)
+
+
+class Kafka(Plugin):
+    """Detect Kafka daemons and sets up configuration to monitor them.
+        This plugin configures the kafka_consumer plugin and does not configure any jmx based checks against kafka.
+        Note this plugin will pull the same information from kafka on each node in the cluster it runs on.
+    """
+
+    def _detect(self):
+        """Run detection, set self.available True if the service is detected."""
+        if find_process_cmdline('kafka') is not None:
+            self.available = True
+
+    def build_config(self):
+        """Build the config as a Plugins object and return.
+        """
+        config = agent_config.Plugins()
+        # First watch the process
+        config.update(watch_process(['kafka']))
+        log.info("\tWatching the kafka process.")
+
+        if self.dependencies_installed():
+            # todo this naively assumes zookeeper is also available on localhost
+
+            import kazoo
+            from kazoo.client import KazooClient
+            logging.getLogger('kazoo').setLevel(logging.WARN)  # kazoo fills up the console without this
+
+            zk = KazooClient(hosts='127.0.0.1:2181', read_only=True)
+            zk.start()
+            topics = {}
+            for topic in zk.get_children('/brokers/topics'):
+                topics[topic] = zk.get_children('/brokers/topics/%s/partitions' % topic)
+
+            consumers = collections.defaultdict(dict)  # {'consumer_group_name': { 'topic1': [ 0, 1, 2] # partitions }}
+            for consumer in zk.get_children('/consumers'):
+                try:
+                    for topic in zk.get_children('/consumers/%s/offsets' % consumer):
+                        if topic in topics:
+                            consumers[consumer][topic] = topics[topic]
+                except kazoo.exceptions.NoNodeError:
+                    continue
+
+
+            log.info("\tInstalling kafka_consumer plugin.")
+            config['kafka_consumer'] = {'init_config': None,
+                                        'instances': [{'kafka_connect_str': 'localhost:9092',
+                                                       'zk_connect_str': 'localhost:2181',
+                                                       'consumer_groups': dict(consumers)}]}
+        return config
+
+    def dependencies_installed(self):
+        try:
+            import kafka
+            import kazoo
+        except ImportError:
+            return False
+
+        return True

monsetup/detection/mysql.py

@@ -8,7 +8,10 @@ log = logging.getLogger(__name__)
 
 class MySQL(Plugin):
     """Detect MySQL daemons and setup configuration to monitor them.
-        This plugin needs user/pass infor for mysql setup, this is best placed in /root/.mysql.cnf
+        This plugin needs user/pass infor for mysql setup, this is best placed in /root/.my.cnf in a format such as
+        [client]
+            user = root
+            password = yourpassword
     """
 
     def _detect(self):
@@ -24,18 +27,18 @@ class MySQL(Plugin):
         config.update(watch_process(['mysqld']))
         log.info("\tWatching the mysqld process.")
 
-        # Attempt login, requires either an empty root password from localhost or relying on a configured .mysql.cnf
+        # Attempt login, requires either an empty root password from localhost or relying on a configured .my.cnf
         if self.dependencies_installed():  # ensures MySQLdb is available
             import MySQLdb
             import _mysql_exceptions
             try:
-                MySQLdb.connect(read_default_file='/root/.mysql.cnf')
+                MySQLdb.connect(read_default_file='/root/.my.cnf')
             except _mysql_exceptions.MySQLError:
                 pass
             else:
-                log.info("\tConfiguring MySQL plugin to connect with auth settings from /root/.mysql.cnf")
+                log.info("\tConfiguring MySQL plugin to connect with auth settings from /root/.my.cnf")
                 config['mysql'] = {'init_config': None, 'instances':
-                                   [{'server': 'localhost', 'user': 'root', 'defaults_file': '/root/.mysql.cnf'}]}
+                                   [{'server': 'localhost', 'user': 'root', 'defaults_file': '/root/.my.cnf'}]}
 
             if not 'mysql' in config:
                 try:

monsetup/detection/zookeeper.py

@@ -0,0 +1,36 @@
+import logging
+import os
+import yaml
+
+from . import Plugin, find_process_cmdline, watch_process
+from monsetup import agent_config
+
+log = logging.getLogger(__name__)
+
+
+class Zookeeper(Plugin):
+    """Detect Zookeeper daemons and setup configuration to monitor them.
+    """
+
+    def _detect(self):
+        """Run detection, set self.available True if the service is detected."""
+        if find_process_cmdline('zookeeper') is not None:
+            self.available = True
+
+    def build_config(self):
+        """Build the config as a Plugins object and return.
+        """
+        config = agent_config.Plugins()
+        # First watch the process
+        log.info("\tWatching the zookeeper process.")
+        config.update(watch_process(['zookeeper']))
+
+        log.info("\tEnabling the zookeeper plugin")
+        with open(os.path.join(self.template_dir, 'conf.d/zk.yaml.example'), 'r') as zk_template:
+            zk_config = yaml.load(zk_template.read())
+        config['zk'] = zk_config
+
+        return config
+
+    def dependencies_installed(self):
+        return True  # The current plugin just does a simple socket connection to zookeeper and parses the stat command

monsetup/main.py

@@ -12,11 +12,11 @@ import sys
 import yaml
 
 import agent_config
-from detection import mysql, network, nova
+from detection import kafka, mysql, network, nova, zookeeper
 from service import sysv
 
 # List of all detection plugins to run
-DETECTION_PLUGINS = [mysql.MySQL, network.Network, nova.Nova]
+DETECTION_PLUGINS = [kafka.Kafka, mysql.MySQL, network.Network, nova.Nova, zookeeper.Zookeeper]
 # Map OS to service type
 OS_SERVICE_MAP = {'linux': sysv.SysV}