Let healthmanager process shutdown cleanly (again)

Change-Id: I5e341915e6e92e5f8279c7c643374ea3d581841b

octavia/amphorae/drivers/health/heartbeat_udp.py

@@ -82,6 +82,7 @@ class UDPStatusGetter(object):
             if self.sock is not None:
                 self.sock.close()
             self.sock = socket.socket(ai_family, socket.SOCK_DGRAM)
+            self.sock.settimeout(1)
             self.sock.bind(self.sockaddr)
             if cfg.CONF.health_manager.sock_rlimit > 0:
                 rlimit = cfg.CONF.health_manager.sock_rlimit
@@ -197,6 +198,9 @@ class UDPStatusGetter(object):
     def check(self):
         try:
             obj, srcaddr = self.dorecv()
+        except socket.timeout:
+            # Pass here as this is an expected cycling of the listen socket
+            pass
         except exceptions.InvalidHMACException:
             # Pass here as the packet was dropped and logged already
             pass

octavia/cmd/health_manager.py

@@ -43,6 +43,9 @@ def hm_listener(exit_event):
         except Exception as e:
             LOG.error('Health Manager listener experienced unknown error: %s',
                       e)
+    LOG.info('Waiting for executor to shutdown...')
+    udp_getter.executor.shutdown()
+    LOG.info('Executor shutdown finished.')
 
 
 def hm_health_check(exit_event):
@@ -93,7 +96,7 @@ def main():
         exit_event.set()
         os.kill(hm_health_check_proc.pid, signal.SIGINT)
         hm_health_check_proc.join()
-        hm_listener_proc.terminate()
+        hm_listener_proc.join()
 
     signal.signal(signal.SIGTERM, process_cleanup)
 

octavia/tests/unit/cmd/test_health_manager.py

@@ -89,8 +89,7 @@ class TestHealthManagerCMD(base.TestCase):
 
         mock_listener_proc.start.assert_called_once_with()
         mock_health_proc.start.assert_called_once_with()
-        mock_listener_proc.join.assert_called_once_with()
+        self.assertEqual(2, mock_listener_proc.join.call_count)
         mock_health_proc.join.assert_called_once_with()
-        mock_listener_proc.terminate.assert_called_once_with()
         mock_kill.assert_called_once_with(mock_health_proc.pid,
                                           signal.SIGINT)