From 3147e5db841f8c3f7efebad055001e823061a50f Mon Sep 17 00:00:00 2001
From: Clint Byrum <clint@fewbar.com>
Date: Fri, 25 Apr 2014 12:34:14 -0700
Subject: [PATCH] Refactor nagios3 element to refresh more often

Previously the element would only refresh from Nova when the
Heat configuration was changed. But we want it to refresh
whenever nova changes.

Note that we won't actually remove hosts that go away.

Change-Id: Ic8af7f30ccc0cc2c64643081a6e300a354d8ad35
---
 elements/nagios3/README.md                    |  3 +-
 elements/nagios3/bin/refresh-nagios-from-nova | 90 +++++++++++++++++++
 elements/nagios3/element-deps                 |  1 -
 elements/nagios3/install.d/17-nagios          | 10 ++-
 elements/nagios3/install.d/nagios3-crontab    |  2 +
 .../etc/nagios3/nova_access_info              |  2 +-
 .../post-configure.d/17-nagios                | 25 +-----
 7 files changed, 105 insertions(+), 28 deletions(-)
 create mode 100755 elements/nagios3/bin/refresh-nagios-from-nova
 create mode 100644 elements/nagios3/install.d/nagios3-crontab

diff --git a/elements/nagios3/README.md b/elements/nagios3/README.md
index 37fd9450e..d35dc7312 100644
--- a/elements/nagios3/README.md
+++ b/elements/nagios3/README.md
@@ -17,7 +17,8 @@ can be provided via heat. For example:
         os_password: unset
         os_username: admin
         os_tenant_name: admin
-        initial_network_split_key: ctlplane
+        monitor_networks:
+          - ctlplane
 
 Sample heat template with defaults configured for a boot-stack vm located at:
   https://git.openstack.org/cgit/openstack/tripleo-heat-templates
diff --git a/elements/nagios3/bin/refresh-nagios-from-nova b/elements/nagios3/bin/refresh-nagios-from-nova
new file mode 100755
index 000000000..02756b627
--- /dev/null
+++ b/elements/nagios3/bin/refresh-nagios-from-nova
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import subprocess
+import sys
+
+from novaclient.v1_1 import client
+from pynag import Model
+
+conn = client.Client(os.environ['OS_USERNAME'],
+                     os.environ['OS_PASSWORD'],
+                     os.environ['OS_TENANT_NAME'],
+                     os.environ['OS_AUTH_URL'])
+
+if len(sys.argv):
+    target_nets = [unicode(x) for x in sys.argv[1:]]
+else:
+    target_net = [u'default-net']
+
+errors = 0
+reload_nagios = False
+
+for net in target_nets:
+    try:
+        Model.Hostgroup.objects.get_by_name(net)
+    except Exception as e:
+        sys.stderr.write("INFO: creating hostgroup %s\n" % net)
+        hg = Model.Hostgroup()
+        hg.name = net
+        hg.hostgroup_name = net
+        if hg.is_dirty():
+            try:
+                if hg.save():
+                    reload_nagios = True
+            except Exception as e:
+                sys.stderr.write("ERROR: %s\n" % e)
+                errors += 1
+                print(hg)
+
+for server in conn.servers.list():
+    server.get()
+    monitor_nets = set(target_nets) & set(server.networks.keys())
+    if not monitor_nets:
+        continue
+    # Prefer first net, and IPv6
+    for net in target_nets:
+        if net in server.networks:
+            addrs = server.networks[net]
+            break
+    # If there is a floating IP, thats what we want.
+    addr = addrs[-1]
+    try:
+        host = Model.Host.objects.get_by_name(server.name)
+    except Exception as e:
+        host = Model.Host()
+        sys.stderr.write("INFO: creating host %s\n" % server.name)
+    host.use = 'generic-host'
+    host.name = server.name
+    host.host_name = server.name
+    host.alias = server.name
+    host.hostgroups = '%s,ssh-servers' % (','.join(monitor_nets),)
+    host.address = addr
+    try:
+        if host.save():
+            reload_nagios = True
+    except Exception as e:
+        sys.stderr.write("ERROR: %s\n" % e)
+        errors += 1
+        print(host)
+
+if errors:
+    sys.stderr.write("ERROR: %d error(s).\n" % errors)
+    sys.exit(1)
+
+if reload_nagios:
+    sys.stderr.write("INFO: reloading nagios.")
+    subprocess.call(["service","nagios3","reload"])
diff --git a/elements/nagios3/element-deps b/elements/nagios3/element-deps
index fceae001c..583462c41 100644
--- a/elements/nagios3/element-deps
+++ b/elements/nagios3/element-deps
@@ -1,5 +1,4 @@
 apache2
-openstack-clients
 os-apply-config
 os-refresh-config
 postfix
diff --git a/elements/nagios3/install.d/17-nagios b/elements/nagios3/install.d/17-nagios
index e50785f97..5f39387bd 100755
--- a/elements/nagios3/install.d/17-nagios
+++ b/elements/nagios3/install.d/17-nagios
@@ -9,8 +9,14 @@ sed -i "s/check_external_commands=0/check_external_commands=1/" /etc/nagios3/nag
 if [ -f /etc/nagios3/conf.d/extinfo_nagios2.cfg ]; then
   sed -i "s/base\/debian/debian/g" /etc/nagios3/conf.d/extinfo_nagios2.cfg
 fi
-# set all servers as having ssh
-sed -i '0,/ssh-servers/! s/localhost/*/' /etc/nagios3/conf.d/hostgroups_nagios2.cfg
+mkdir -p /etc/nagios3/pynag
+if ! grep -q '^cfg_dir=/etc/nagios3/pynag' /etc/nagios3/nagios.cfg ; then
+    echo "cfg_dir=/etc/nagios3/pynag" >> /etc/nagios3/nagios.cfg
+fi
+install -m 0755 -o root -g root $(dirname $0)/../bin/refresh-nagios-from-nova /usr/local/bin
+install -m 0644 -o root -g root $(dirname $0)/nagios3-crontab /etc/cron.d/nagios3
+virtualenv /opt/stack/venvs/nagios-tools
+/opt/stack/venvs/nagios-tools/bin/pip install pynag python-novaclient
 # some cleanup
 chmod g+x /var/lib/nagios3/rw
 chmod g+x /var/lib/nagios3
diff --git a/elements/nagios3/install.d/nagios3-crontab b/elements/nagios3/install.d/nagios3-crontab
new file mode 100644
index 000000000..4cb62301f
--- /dev/null
+++ b/elements/nagios3/install.d/nagios3-crontab
@@ -0,0 +1,2 @@
+# Refresh from nova periodically
+*/5 * * * * root bash -c '. /opt/stack/venvs/nagios-tools/bin/activate ; . /etc/nagios3/nova_access_info ; /usr/local/bin/refresh-nagios-from-nova $monitor_networks'
diff --git a/elements/nagios3/os-apply-config/etc/nagios3/nova_access_info b/elements/nagios3/os-apply-config/etc/nagios3/nova_access_info
index c5c918073..e0f12720b 100644
--- a/elements/nagios3/os-apply-config/etc/nagios3/nova_access_info
+++ b/elements/nagios3/os-apply-config/etc/nagios3/nova_access_info
@@ -6,5 +6,5 @@ export OS_TENANT_NAME={{nagios3.os_tenant_name}}
 export COMPUTE_API_VERSION=1.1
 export OS_NO_CACHE=True
 # add initial split network.
-split_key={{nagios3.initial_network_split_key}}
+monitor_networks="{{#nagios3.monitor_networks}}{{.}} {{/nagios3.monitor_networks}}"
 nagiosadmin_pass={{nagios3.adm_web_passwd}}
diff --git a/elements/nagios3/os-refresh-config/post-configure.d/17-nagios b/elements/nagios3/os-refresh-config/post-configure.d/17-nagios
index dc2225095..dbefd52a1 100755
--- a/elements/nagios3/os-refresh-config/post-configure.d/17-nagios
+++ b/elements/nagios3/os-refresh-config/post-configure.d/17-nagios
@@ -1,28 +1,7 @@
 #!/bin/bash
-set -eux
+set -eu
 
-# exit if we cann't source our required data
+# exit if we can't source our required data
 DEFAULTS=/etc/nagios3/nova_access_info
-[ -e $DEFAULTS ] || exit 0
 source $DEFAULTS
-echo "defaults loaded."
 htpasswd -bc /etc/nagios3/htpasswd.users nagiosadmin $nagiosadmin_pass
-nova_ip_list=$(nova list | awk -F"|" '/'$split_key'/ { split($7,x,"="); print x[2] }')
-echo "Found $nova_ip_list"
-for node_ip in $nova_ip_list
-do
-    echo "setup cfg file for $node_ip."
-    # TODO: make host_name & alias real names
-    if [ ! -f /etc/nagios3/conf.d/$node_ip.cfg ]; then
-        cat << _EOF_ >> /etc/nagios3/conf.d/$node_ip.cfg
-define host{
-        use                     generic-host
-        host_name               $node_ip
-        alias                   $node_ip
-        address                 $node_ip
-        }
-_EOF_
-    fi
-done
-
-service nagios3 restart