Browse Source

Finish changes to stop an engine

Added a call to stop all react scripts, by passing all known routing
keys to the react_killer function. Then stop the threadpool executor.
Finally, Raise a known exception to stop the watchdog thread. This
will throw an ugly traceback, but will shutdown the engine
gracefully.

Also made minor changes to the example react.json, to change the log
format. Knowing the time a log was printed is useful.

Change-Id: Ibed06f79547312d188feb499f937eb5390d60c3e
Pranesh Pandurangan 4 years ago
parent
commit
c79d12c645
4 changed files with 31 additions and 18 deletions
  1. 1
    9
      entropy/__main__.py
  2. 28
    8
      entropy/engine.py
  3. 1
    1
      entropy/examples/repair/react.json
  4. 1
    0
      entropy/utils.py

+ 1
- 9
entropy/__main__.py View File

@@ -18,9 +18,7 @@
18 18
 import argparse
19 19
 import logging
20 20
 import os
21
-import psutil
22 21
 import tempfile
23
-import time
24 22
 
25 23
 from engine import Engine
26 24
 from entropy import utils
@@ -152,13 +150,7 @@ def start_engine(args):
152 150
 def stop_engine(args):
153 151
     LOG.info("Stopping engine %s", args.name)
154 152
     # Grab engine config file, set our engine to disabled
155
-    pid = utils.disable_engine(args.name, engine_cfg)
156
-    try:
157
-        p = psutil.Process(pid)
158
-        time.sleep(5)
159
-        p.terminate()
160
-    except psutil.NoSuchProcess:
161
-        LOG.exception("No running engine %s", args.name)
153
+    utils.disable_engine(args.name, engine_cfg)
162 154
 
163 155
 
164 156
 def parse():

+ 28
- 8
entropy/engine.py View File

@@ -73,8 +73,11 @@ class Engine(object):
73 73
         self.futures = []
74 74
         self.run_queue = collections.deque()
75 75
         # Private variables
76
-        self._watchdog_event_fn = {self.repair_cfg: self.repair_modified,
77
-                                   self.engine_cfg: self.engine_disabled}
76
+        self._watchdog_event_fn = {
77
+            self.repair_cfg: self.repair_modified,
78
+            self.engine_cfg: self.engine_disabled,
79
+            self.audit_cfg: self.audit_modified,
80
+        }
78 81
         # Private variables to keep track of repair scripts.
79 82
         self._repairs = []
80 83
         self._known_routing_keys = collections.defaultdict(list)
@@ -126,6 +129,7 @@ class Engine(object):
126 129
         self.start_scheduler()
127 130
 
128 131
     def start_scheduler(self):
132
+        # Start serializer
129 133
         if not self._serializer:
130 134
             self._serializer = self.executor.submit(self.start_serializer)
131 135
             self.futures.append(self._serializer)
@@ -137,6 +141,7 @@ class Engine(object):
137 141
         self.futures.extend(self.start_react_scripts(
138 142
             self._get_react_scripts()))
139 143
 
144
+        # Start scheduler
140 145
         self._scheduler = self.executor.submit(self.schedule)
141 146
         self.futures.append(self._scheduler)
142 147
         self._scheduler.add_done_callback(
@@ -257,15 +262,32 @@ class Engine(object):
257 262
 
258 263
     def stop_engine(self):
259 264
         LOG.info("Stopping engine %s", self.name)
260
-        # Set state to stop, which will stop serializers
265
+        # Set state to stop, which will stop serializer, and scheduler
261 266
         self._state = states.DISABLED
262 267
         # Clear run queue
263 268
         LOG.info("Clearing audit run queue for %s", self.name)
264 269
         self.run_queue.clear()
265
-        # Stop all repairs - not yet implemented
270
+
271
+        # Stop all repairs
272
+        LOG.info("Stopping all repairs for %s", self.name)
273
+        repairs_to_stop = self._known_routing_keys.keys()
274
+        self.stop_react_scripts(repairs_to_stop)
275
+
276
+        # Stop the executor - this is a blocking call.
277
+        LOG.info("Shutting down executor for %s", self.name)
278
+        self.executor.shutdown()
279
+
266 280
         # Stop watchdog monitoring
267 281
         LOG.info("Stopping watchdog for %s", self.name)
268
-        self._watchdog_thread.stop()
282
+        # NOTE(praneshp): Till the watchdog authors respond with the right way
283
+        # to stop watchdog, we'll raise something from here. That will stop
284
+        # the watchdog thread, go back to the join() in start_scheduler(), and
285
+        # quit the program
286
+        raise exceptions.EngineStoppedException(
287
+            'Fake exception to kill watchdog thread')
288
+
289
+    def audit_modified(self):
290
+        return NotImplemented
269 291
 
270 292
     def repair_modified(self):
271 293
         LOG.info('Repair configuration changed')
@@ -289,7 +311,7 @@ class Engine(object):
289 311
     def start_watchdog(self):
290 312
         LOG.debug('Watchdog mapping is: ', self._watchdog_event_fn)
291 313
         dirs_to_watch = [utils.get_filename_and_path(x)[0] for x in
292
-                         self.engine_cfg, self.repair_cfg]
314
+                         self.engine_cfg, self.repair_cfg, self.audit_cfg]
293 315
         return utils.watch_dir_for_change(dirs_to_watch,
294 316
                                           self._watchdog_event_fn)
295 317
 
@@ -323,11 +345,9 @@ class Engine(object):
323 345
 
324 346
     def stop_react_scripts(self, repairs_to_stop):
325 347
         # current react scripts
326
-        LOG.info("Currently running react scripts: %s", self._repairs)
327 348
         for repair in repairs_to_stop:
328 349
             self.stop_react(repair)
329 350
         # react scripts at the end
330
-        LOG.info("Currently running react scripts: %s", self._repairs)
331 351
 
332 352
     def stop_react(self, repair):
333 353
         LOG.info("Stopping react script %s", repair)

+ 1
- 1
entropy/examples/repair/react.json View File

@@ -9,5 +9,5 @@
9 9
     "mq_password": "guest",
10 10
     "routing_key": "demo",
11 11
     "log_file": "/home/praneshp/code/entropy/entropy/examples/logs/demo.log",
12
-    "log_format": "%(filename)s %(lineno)s %(message)s"
12
+    "log_format": "%(asctime)s %(lineno)s %(message)s"
13 13
 }

+ 1
- 0
entropy/utils.py View File

@@ -110,6 +110,7 @@ def watch_dir_for_change(dirs_to_watch, event_fn):
110 110
     observer = Observer()
111 111
     for directory in dirs_to_watch:
112 112
         observer.schedule(event_handler, path=directory)
113
+    observer.setDaemon(True)
113 114
     observer.start()
114 115
     return observer
115 116
 

Loading…
Cancel
Save