Refactor nagios3 element to refresh more often

Previously the element would only refresh from Nova when the
Heat configuration was changed. But we want it to refresh
whenever nova changes.

Note that we won't actually remove hosts that go away.

Change-Id: Ic8af7f30ccc0cc2c64643081a6e300a354d8ad35

elements/nagios3/README.md

@@ -17,7 +17,8 @@ can be provided via heat. For example:
         os_password: unset
         os_username: admin
         os_tenant_name: admin
-        initial_network_split_key: ctlplane
+        monitor_networks:
+          - ctlplane
 
 Sample heat template with defaults configured for a boot-stack vm located at:
   https://git.openstack.org/cgit/openstack/tripleo-heat-templates

elements/nagios3/bin/refresh-nagios-from-nova

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import subprocess
+import sys
+
+from novaclient.v1_1 import client
+from pynag import Model
+
+conn = client.Client(os.environ['OS_USERNAME'],
+                     os.environ['OS_PASSWORD'],
+                     os.environ['OS_TENANT_NAME'],
+                     os.environ['OS_AUTH_URL'])
+
+if len(sys.argv):
+    target_nets = [unicode(x) for x in sys.argv[1:]]
+else:
+    target_net = [u'default-net']
+
+errors = 0
+reload_nagios = False
+
+for net in target_nets:
+    try:
+        Model.Hostgroup.objects.get_by_name(net)
+    except Exception as e:
+        sys.stderr.write("INFO: creating hostgroup %s\n" % net)
+        hg = Model.Hostgroup()
+        hg.name = net
+        hg.hostgroup_name = net
+        if hg.is_dirty():
+            try:
+                if hg.save():
+                    reload_nagios = True
+            except Exception as e:
+                sys.stderr.write("ERROR: %s\n" % e)
+                errors += 1
+                print(hg)
+
+for server in conn.servers.list():
+    server.get()
+    monitor_nets = set(target_nets) & set(server.networks.keys())
+    if not monitor_nets:
+        continue
+    # Prefer first net, and IPv6
+    for net in target_nets:
+        if net in server.networks:
+            addrs = server.networks[net]
+            break
+    # If there is a floating IP, thats what we want.
+    addr = addrs[-1]
+    try:
+        host = Model.Host.objects.get_by_name(server.name)
+    except Exception as e:
+        host = Model.Host()
+        sys.stderr.write("INFO: creating host %s\n" % server.name)
+    host.use = 'generic-host'
+    host.name = server.name
+    host.host_name = server.name
+    host.alias = server.name
+    host.hostgroups = '%s,ssh-servers' % (','.join(monitor_nets),)
+    host.address = addr
+    try:
+        if host.save():
+            reload_nagios = True
+    except Exception as e:
+        sys.stderr.write("ERROR: %s\n" % e)
+        errors += 1
+        print(host)
+
+if errors:
+    sys.stderr.write("ERROR: %d error(s).\n" % errors)
+    sys.exit(1)
+
+if reload_nagios:
+    sys.stderr.write("INFO: reloading nagios.")
+    subprocess.call(["service","nagios3","reload"])

elements/nagios3/element-deps

@@ -1,5 +1,4 @@
 apache2
-openstack-clients
 os-apply-config
 os-refresh-config
 postfix

elements/nagios3/install.d/17-nagios

@@ -9,8 +9,14 @@ sed -i "s/check_external_commands=0/check_external_commands=1/" /etc/nagios3/nag
 if [ -f /etc/nagios3/conf.d/extinfo_nagios2.cfg ]; then
   sed -i "s/base\/debian/debian/g" /etc/nagios3/conf.d/extinfo_nagios2.cfg
 fi
-# set all servers as having ssh
-sed -i '0,/ssh-servers/! s/localhost/*/' /etc/nagios3/conf.d/hostgroups_nagios2.cfg
+mkdir -p /etc/nagios3/pynag
+if ! grep -q '^cfg_dir=/etc/nagios3/pynag' /etc/nagios3/nagios.cfg ; then
+    echo "cfg_dir=/etc/nagios3/pynag" >> /etc/nagios3/nagios.cfg
+fi
+install -m 0755 -o root -g root $(dirname $0)/../bin/refresh-nagios-from-nova /usr/local/bin
+install -m 0644 -o root -g root $(dirname $0)/nagios3-crontab /etc/cron.d/nagios3
+virtualenv /opt/stack/venvs/nagios-tools
+/opt/stack/venvs/nagios-tools/bin/pip install pynag python-novaclient
 # some cleanup
 chmod g+x /var/lib/nagios3/rw
 chmod g+x /var/lib/nagios3

elements/nagios3/install.d/nagios3-crontab

@@ -0,0 +1,2 @@
+# Refresh from nova periodically
+*/5 * * * * root bash -c '. /opt/stack/venvs/nagios-tools/bin/activate ; . /etc/nagios3/nova_access_info ; /usr/local/bin/refresh-nagios-from-nova $monitor_networks'

elements/nagios3/os-apply-config/etc/nagios3/nova_access_info

@@ -6,5 +6,5 @@ export OS_TENANT_NAME={{nagios3.os_tenant_name}}
 export COMPUTE_API_VERSION=1.1
 export OS_NO_CACHE=True
 # add initial split network.
-split_key={{nagios3.initial_network_split_key}}
+monitor_networks="{{#nagios3.monitor_networks}}{{.}} {{/nagios3.monitor_networks}}"
 nagiosadmin_pass={{nagios3.adm_web_passwd}}

elements/nagios3/os-refresh-config/post-configure.d/17-nagios

@@ -1,28 +1,7 @@
 #!/bin/bash
-set -eux
+set -eu
 
-# exit if we cann't source our required data
+# exit if we can't source our required data
 DEFAULTS=/etc/nagios3/nova_access_info
-[ -e $DEFAULTS ] || exit 0
 source $DEFAULTS
-echo "defaults loaded."
 htpasswd -bc /etc/nagios3/htpasswd.users nagiosadmin $nagiosadmin_pass
-nova_ip_list=$(nova list | awk -F"|" '/'$split_key'/ { split($7,x,"="); print x[2] }')
-echo "Found $nova_ip_list"
-for node_ip in $nova_ip_list
-do
-    echo "setup cfg file for $node_ip."
-    # TODO: make host_name & alias real names
-    if [ ! -f /etc/nagios3/conf.d/$node_ip.cfg ]; then
-        cat << _EOF_ >> /etc/nagios3/conf.d/$node_ip.cfg
-define host{
-        use                     generic-host
-        host_name               $node_ip
-        alias                   $node_ip
-        address                 $node_ip
-        }
-_EOF_
-    fi
-done
-
-service nagios3 restart