Temporarily override Ansible linear strategy (2/2)

The log streaming callback is not being called in the same way
in Ansible 2.5 as it was in 2.3.  In particular, in some cases
different Task objects are used for different hosts.  This,
combined with the fact that the callback is only called once for
a given task means that in these cases we are unable to supply
the zuul_log_id to the Task object for the second host on a task.

To correct this, a local copy of the linear strategy plugin is
added, with the change that for every host-task, it calls either
the normal on_task_start callback, or a new zuul_task_start
callback.  This ensures that we are able to set up log streaming
on every host-task.

We plan to move to a different system for establishing log streaming
soon so that we don't have to keep carrying this patched plugin.

Story: 2002528
Task: 22067

Change-Id: Ifd17d799fc28174df5194a6cd09e0e25f3ea75ac
Co-Authored-By: Tobias Henkel <tobias.henkel@bmw-carit.de>
Co-Authored-By: Clark Boylan <clark.boylan@gmail.com>

zuul/ansible/callback/zuul_stream.py

@@ -236,6 +236,11 @@ class CallbackModule(default.CallbackModule):
                 streamer.start()
                 self._streamers.append(streamer)
 
+    def zuul_on_task_start(self, task, is_conditional):
+        if (task.action in ('command', 'shell') and
+            'zuul_log_id' not in task.args):
+            self.v2_playbook_on_task_start(task, False)
+
     def v2_playbook_on_handler_task_start(self, task):
         self.v2_playbook_on_task_start(task, False)
 

zuul/ansible/paths.py

@@ -20,6 +20,7 @@ from ansible.errors import AnsibleError
 import ansible.modules
 import ansible.plugins.action
 import ansible.plugins.lookup
+import ansible.plugins.strategy
 
 
 def _safe_find_needle(super, dirname, needle):
@@ -128,6 +129,14 @@ def _import_ansible_lookup_plugin(name):
         *imp.find_module(name, ansible.plugins.lookup.__path__))
 
 
+def _import_ansible_strategy_plugin(name):
+    # See _import_ansible_action_plugin
+
+    return imp.load_module(
+        'zuul.ansible.protected.lookup.' + name,
+        *imp.find_module(name, ansible.plugins.strategy.__path__))
+
+
 def _is_official_module(module):
     task_module_path = module._shared_loader_obj.module_loader.find_plugin(
         module._task.action)

zuul/ansible/strategy/linear.py

@@ -290,22 +290,25 @@ class StrategyModule(StrategyBase):
                         if (task.any_errors_fatal or run_once) and not task.ignore_errors:
                             any_errors_fatal = True
 
+                        display.debug("sending task start callback, copying the task so we can template it temporarily")
+                        saved_name = task.name
+                        display.debug("done copying, going to template now")
+                        try:
+                            task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
+                            display.debug("done templating")
+                        except:
+                            # just ignore any errors during task name templating,
+                            # we don't care if it just shows the raw name
+                            display.debug("templating failed for some reason")
+                        display.debug("here goes the callback...")
                         if not callback_sent:
-                            display.debug("sending task start callback, copying the task so we can template it temporarily")
-                            saved_name = task.name
-                            display.debug("done copying, going to template now")
-                            try:
-                                task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty')
-                                display.debug("done templating")
-                            except:
-                                # just ignore any errors during task name templating,
-                                # we don't care if it just shows the raw name
-                                display.debug("templating failed for some reason")
-                            display.debug("here goes the callback...")
                             self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False)
-                            task.name = saved_name
+                        else:
-                            callback_sent = True
+                            self._tqm.send_callback('zuul_on_task_start', task, is_conditional=False)
-                            display.debug("sending task start callback")
+
+                        task.name = saved_name
+                        callback_sent = True
+                        display.debug("sending task start callback")
 
                         self._blocked_hosts[host.get_name()] = True
                         self._queue_task(host, task, task_vars, play_context)

zuul/executor/server.py

@@ -1401,6 +1401,8 @@ class AnsibleJob(object):
                          % self.executor_server.library_dir)
             config.write('command_warnings = False\n')
             config.write('callback_plugins = %s\n' % callback_path)
+            config.write('strategy_plugins = %s\n' %
+                         self.executor_server.strategy_dir)
             config.write('stdout_callback = zuul_stream\n')
             config.write('filter_plugins = %s\n'
                          % self.executor_server.filter_dir)
@@ -1856,6 +1858,7 @@ class ExecutorServer(object):
         self.library_dir = os.path.join(plugin_dir, 'library')
         self.action_dir = os.path.join(plugin_dir, 'action')
         self.callback_dir = os.path.join(plugin_dir, 'callback')
+        self.strategy_dir = os.path.join(plugin_dir, 'strategy')
         self.lookup_dir = os.path.join(plugin_dir, 'lookup')
         self.filter_dir = os.path.join(plugin_dir, 'filter')