Merge "Improve snmpd element."

elements/snmpd/README.md

@@ -1,13 +1,22 @@
 Simple Network Management Protocol (SNMP) daemon.
 
-Installs packaged snmpd, snmp-mibs-downloader, and lm-sensors. Creates a
-read only user with password for snmp v3 authentication.
+Installs packaged snmpd, snmp-mibs-downloader, lm-sensors and
+nagios-plugins-basic.
 
-Enabling the SNMP daemon allows more advanced monitoring of the deployed nodes.
+nagios-plugins-basic is installed to provide a basic frame work for the initial
+check.
 
+Nagios style check for open file limits, per process is provided.
+
+Creates a read only user with password for snmp v3 authentication.
+exposes OoOEnabled and CheckProcOpenFiles via the NET-SNMP-EXTEND-MIB OID
+block.
+
+Grants snmp user password-less sudo access to lsof, so that the per process
+check works correctly.
 
 Options should be provided via heat. For example:
-    snmp:
-        MIB: UCD-SNMP-MIB
+    snmpd:
+        export_MIB: UCD-SNMP-MIB
         readonly_user_name: RoUser
         readonly_user_password: password

elements/snmpd/files/check_proc_open_files.py

@@ -0,0 +1,93 @@
+#! /usr/bin/env python
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import shlex
+import subprocess
+import sys
+import itertools
+
+warn_limit = 90
+crit_limit = 95
+crit_msg = ''
+warn_msg = ''
+output_msg = ''
+status = {'OK': 0, 'WARNING': 1, 'CRITICAL': 2, 'UNKNOWN': 3}
+
+
+def _get_open_file_limits(file_handle):
+    limit = 0
+    for line in file_handle:
+        if line.startswith('Max open files'):
+            limit = line.split()[3]
+    return limit
+
+
+# Define an in-line generator, no temporary lists
+def _split_lines(lines):
+    for line in lines:
+        yield line[0:19].strip().split()
+
+
+def _get_lsof_pid_list():
+    # get open files limit per PID exclude any line with a TID.
+    lsof_cmd = shlex.split("sudo lsof -n")
+    lsof_proc = subprocess.Popen(lsof_cmd, stdout=subprocess.PIPE)
+    filtered_lines = itertools.ifilterfalse(lambda line: line[1] == 'PID',
+                                            _split_lines(lsof_proc.stdout))
+    filtered_by_len = itertools.ifilter(lambda line: len(line) == 2,
+                                        filtered_lines)
+    for cmd, pid in itertools.groupby(filtered_by_len, lambda line: line[:2]):
+        yield cmd, pid
+
+
+for (pid_cmd, pid_number), count in _get_lsof_pid_list():
+    pid_count = len(list(count))
+    pid_limit_file = "/proc/%s/limits" % pid_number
+    try:
+        with open(pid_limit_file, 'rb') as limit_file_handle:
+            pid_limit = _get_open_file_limits(limit_file_handle)
+    except IOError:
+        # no limit file skip
+        continue
+
+    pid_usage_pct = pid_count * 100 / int(pid_limit)
+    if pid_usage_pct >= crit_limit:
+        crit_msg += ": %s/%s/%s %s" % (pid_number,
+                                       pid_count,
+                                       pid_limit,
+                                       pid_cmd)
+    elif pid_usage_pct >= warn_limit and pid_usage_pct < crit_limit:
+        warn_msg += ": %s/%s/%s %s" % (pid_number,
+                                       pid_count,
+                                       pid_limit,
+                                       pid_cmd)
+
+# if any critical services then exit critical (include warnings)
+if crit_msg:
+    output_msg = "Critical: pid/files/limit Proc%s\n%s" % (crit_msg,
+                                                           warn_msg)
+    exit_code = status['CRITICAL']
+# if not critical and warn not empty exit warning
+if warn_msg:
+    output_msg = "Warning: pid/files/limit Proc%s" % warn_msg
+    exit_code = status['WARNING']
+# All ok here
+if not output_msg:
+    output_msg = "OK - All processes open files with in limits."
+    exit_code = status['OK']
+
+print output_msg
+sys.exit(exit_code)

elements/snmpd/files/snmp

@@ -0,0 +1 @@
+snmp ALL=(root) NOPASSWD: /usr/bin/lsof

elements/snmpd/install.d/67-snmpd

@@ -1,4 +1,7 @@
 #!/bin/bash
 set -eux
 
-install-packages snmpd snmp-mibs-downloader lm-sensors
+install-packages snmpd snmp-mibs-downloader lm-sensors nagios-plugins-basic
+# install our snmp check script
+install -m 0755 -o root -g root $(dirname $0)/../files/check_proc_open_files.py /usr/lib/nagios/plugins/check_proc_open_files.py
+install -m 0440 -o root -g root $(dirname $0)/../files/snmp /etc/sudoers.d/snmp

elements/snmpd/os-apply-config/etc/default/snmpd

@@ -1,6 +1,6 @@
 # This file controls the activity of snmpd and snmptrapd
 # enable a default MIB
-export MIBS={{snmp.MIB}}
+export MIBS={{snmpd.export_MIB}}
 
 # snmpd control (yes means start daemon).
 SNMPDRUN=yes

elements/snmpd/os-apply-config/etc/snmp/snmpd.conf

@@ -1,9 +1,9 @@
 #  Listen for connections on all interfaces (both IPv4 *and* IPv6)
 agentAddress udp:161,udp6:[::1]:161
-createUser {{snmp.readonly_user_name}}  MD5 "{{snmp.readonly_user_password}}"
+createUser {{snmpd.readonly_user_name}}  MD5 "{{snmpd.readonly_user_password}}"
 view   systemonly  included   .1.3.6.1.2.1.1
 view   systemonly  included   .1.3.6.1.2.1.25.1
-rouser   authOnlyUser
+rouser   {{snmpd.readonly_user_name}}
 sysLocation    Sitting on top of the cloud
 sysContact     Me <me@example.org>
 
@@ -27,9 +27,8 @@ defaultMonitors          yes
 linkUpDownNotifications  yes
 
 #  Arbitrary extension commands
-
-#extend    test1   /bin/echo  Hello, cloud!
-#extend-sh test2   echo Hello, cloud ; echo Hi there ; exit 35
+extend  OoOEnabled /bin/echo  Yes
+extend  CheckProcOpenFiles /bin/bash /usr/lib/nagios/plugins/check_proc_open_files.py
 
 #  "Pass-through" MIB extension command
 #pass .1.3.6.1.4.1.8072.2.255  /bin/sh       PREFIX/local/passtest