Browse Source

Add callbacks for futures

Added callbacks for the known futures (react scripts, scheduler,
serializer). This will help us debug known long-lasting futures
easily. Define one function called future_callback, and use
functools to pass arguments to the callbacks to determine type
of future, etc

Change-Id: I579a41e92736568535ce3f64c5507ac5224d5a62
Pranesh Pandurangan 4 years ago
parent
commit
857fb52cf1
1 changed files with 27 additions and 2 deletions
  1. 27
    2
      entropy/engine.py

+ 27
- 2
entropy/engine.py View File

@@ -17,6 +17,7 @@
17 17
 
18 18
 import collections
19 19
 import datetime
20
+import functools
20 21
 import logging
21 22
 import operator
22 23
 import os
@@ -83,6 +84,7 @@ class Engine(object):
83 84
 
84 85
         # Serializer related variables
85 86
         self._serializer = None
87
+        self._scheduler = None
86 88
 
87 89
         # State related variables
88 90
         self._state = states.ENABLED
@@ -127,13 +129,19 @@ class Engine(object):
127 129
         if not self._serializer:
128 130
             self._serializer = self.executor.submit(self.start_serializer)
129 131
             self.futures.append(self._serializer)
132
+            self._serializer.add_done_callback(
133
+                functools.partial(self.future_callback,
134
+                                  future_type='serializer'))
130 135
 
131 136
         # Start react scripts.
132 137
         self.futures.extend(self.start_react_scripts(
133 138
             self._get_react_scripts()))
134 139
 
135
-        scheduler = self.executor.submit(self.schedule)
136
-        self.futures.append(scheduler)
140
+        self._scheduler = self.executor.submit(self.schedule)
141
+        self.futures.append(self._scheduler)
142
+        self._scheduler.add_done_callback(
143
+            functools.partial(self.future_callback,
144
+                              future_type='scheduler'))
137 145
 
138 146
         # watchdog
139 147
         self._watchdog_thread = self.start_watchdog()
@@ -147,6 +155,19 @@ class Engine(object):
147 155
             if next_jobs:
148 156
                 self.setup_audit(next_time, next_jobs)
149 157
 
158
+    def future_callback(self, future, future_type='unknown_type',
159
+                        name='no_name', **kwargs):
160
+        if future_type in ['serializer', 'scheduler']:
161
+            LOG.info('Call back for %s, finished state %s',
162
+                     future_type, future.done())
163
+            return
164
+        if future_type == 'react':
165
+            LOG.info('Callback for react script %s', name)
166
+            for future in self._repairs:
167
+                if not future.done():
168
+                    LOG.info('Not all react scripts are done yet')
169
+                    return
170
+
150 171
     def wait_next(self, timeout=None):
151 172
         watch = None
152 173
         next_jobs = []
@@ -377,6 +398,10 @@ class Engine(object):
377 398
             self.running_repairs.append(script)
378 399
             imported_module = imp.load_module(react_script, *available_modules)
379 400
             future = self.executor.submit(imported_module.main, **kwargs)
401
+            future.add_done_callback(
402
+                functools.partial(self.future_callback,
403
+                                  future_type='react',
404
+                                  name=kwargs['name']))
380 405
             self._repairs.append(future)
381 406
             return future
382 407
         except Exception:

Loading…
Cancel
Save