From d565c47fb16a9702286c7d7d5291e7065443d917 Mon Sep 17 00:00:00 2001
From: Kengo Takahara <takahara.kengo.z03@as.ntts.co.jp>
Date: Thu, 9 Feb 2017 19:30:03 +0900
Subject: [PATCH] Add implement of sending notification

This patch adds implementation of sending a notification.

Change-Id: I56f592e9ce91447ac35fb14073c0c94cb9677472
Implements: bp pythonize-host-and-process-monitor
---
 masakarimonitors/conf/process.py              |  8 ++++
 masakarimonitors/processmonitor/process.py    |  3 ++
 .../process_handler/handle_process.py         | 46 ++++++++++++++++++-
 3 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/masakarimonitors/conf/process.py b/masakarimonitors/conf/process.py
index 0e502fb..8474720 100644
--- a/masakarimonitors/conf/process.py
+++ b/masakarimonitors/conf/process.py
@@ -25,6 +25,14 @@ monitor_process_opts = [
     cfg.IntOpt('restart_interval',
                default=5,
                help='Interval in seconds for restarting a process.'),
+    cfg.IntOpt('api_retry_max',
+               default=12,
+               help='Number of retries for send a notification in'
+                    ' processmonitor.'),
+    cfg.IntOpt('api_retry_interval',
+               default=10,
+               help='Interval between re-sending a notification in'
+                    ' processmonitor(in seconds).'),
     cfg.StrOpt('process_list_path',
                default='/etc/masakarimonitors/process_list.yaml',
                help='The file path of process list.'),
diff --git a/masakarimonitors/processmonitor/process.py b/masakarimonitors/processmonitor/process.py
index d575968..74652c9 100644
--- a/masakarimonitors/processmonitor/process.py
+++ b/masakarimonitors/processmonitor/process.py
@@ -74,6 +74,9 @@ class ProcessmonitorManager(manager.Manager):
                 if len(down_process_list) != 0:
                     # Restart down processes.
                     self.process_handler.restart_processes(down_process_list)
+                else:
+                    # Since no down process, clear the restart_failure_list
+                    self.process_handler.restart_failure_list[:] = []
 
                 # Reload process list and set to the process handler.
                 process_list = self._load_process_list()
diff --git a/masakarimonitors/processmonitor/process_handler/handle_process.py b/masakarimonitors/processmonitor/process_handler/handle_process.py
index 0b9992e..d2bd74b 100644
--- a/masakarimonitors/processmonitor/process_handler/handle_process.py
+++ b/masakarimonitors/processmonitor/process_handler/handle_process.py
@@ -12,13 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import socket
+
 import eventlet
 from oslo_log import log as oslo_logging
+from oslo_utils import timeutils
 
 import masakarimonitors.conf
+from masakarimonitors.ha import masakari
 from masakarimonitors.i18n import _LE
 from masakarimonitors.i18n import _LI
 from masakarimonitors.i18n import _LW
+from masakarimonitors.objects import event_constants as ec
 from masakarimonitors import utils
 
 LOG = oslo_logging.getLogger(__name__)
@@ -30,6 +35,8 @@ class HandleProcess(object):
 
     def __init__(self):
         self.process_list = None
+        self.restart_failure_list = []
+        self.notifier = masakari.SendNotification()
 
     def set_process_list(self, process_list):
         """Set process list object.
@@ -106,6 +113,24 @@ class HandleProcess(object):
 
         return down_process_list
 
+    def _make_event(self, process_name):
+
+        hostname = socket.gethostname()
+        current_time = timeutils.utcnow()
+        event = {
+            'notification': {
+                'type': ec.EventConstants.TYPE_PROCESS,
+                'hostname': hostname,
+                'generated_time': current_time,
+                'payload': {
+                    'event': ec.EventConstants.EVENT_STOPPED,
+                    'process_name': process_name
+                }
+            }
+        }
+
+        return event
+
     def restart_processes(self, down_process_list):
         """Restart processes.
 
@@ -114,7 +139,16 @@ class HandleProcess(object):
 
         :param down_process_list: down process list object
         """
+        tmp_restart_failure_list = []
         for down_process in down_process_list:
+            # The process which failed to restart previously doesn't restart.
+            if down_process['process_name'] in self.restart_failure_list:
+                msg = "Process '%s' doesn't be restarted because it failed" \
+                      " to restart previously." % down_process['process_name']
+                LOG.warning(_LW("%s"), msg)
+                tmp_restart_failure_list.append(down_process['process_name'])
+                continue
+
             cmd_str = down_process['restart_command']
 
             LOG.info(
@@ -135,4 +169,14 @@ class HandleProcess(object):
 
             if retries == CONF.process.restart_retries:
                 # Send a notification.
-                pass
+                event = self._make_event(down_process['process_name'])
+                self.notifier.send_notification(
+                    CONF.process.api_retry_max,
+                    CONF.process.api_retry_interval,
+                    event)
+                # Add the process name which failed to restart to the
+                # failure list.
+                tmp_restart_failure_list.append(down_process['process_name'])
+
+        # Replace the old restart_failure_list with new one
+        self.restart_failure_list = tmp_restart_failure_list[:]