Improve zuul_console_disabled testing

This adds a function test that verifies:
* That we really do avoid creating a connection to the log streamer
  when zuul_console_disabled is set.

  This is accomplished by ensuring that we don't get a
  "Starting to log..." message in the ansible-playbook command output.
* That we do include the task output in the console log text file
  when zuul_console_disabled is set (we copy it from the task result
  dictionary).

The existing tests around this functionality did not have effective
assertions for the first item, and did not effectively exercise it
because they set zuul_console_disabled as a task variable.  We expect
it to be set as a host variable (since it is a characteristic of the
host), so the test is updated to use set_fact instead.

(It would be ideal to also handle it as a task variable, but it's not
clear if there is a reasonable way to do that in the callback plugin.)

Change-Id: I780e265b4d86ace7dd06d577a44ed5a61fe2a740

playbooks/zuul-stream/fixtures/test-stream.yaml

@@ -2,13 +2,17 @@
 # if we set zuul_console_disabled, we don't try to connect at all.  If
 # there is a log streamer running when we run this test, then we have
 # no indication that we avoid the connection step.
+- name: Disable zuul console
+  hosts: node
+  tasks:
+    - set_fact:
+        zuul_console_disabled: true
+
 - name: Run command to show skipping works without zuul_console running
-  vars:
-    zuul_console_disabled: true
   hosts: node
   tasks:
     - name: Run quiet command
-      command: echo 'This command should not stream'
+      command: echo 'This command should not stream 1'
       when: new_console | default(false)
 
 - name: Start zuul stream daemon
@@ -25,12 +29,16 @@
       when: new_console | default(false)
 
 - name: Run command to show skipping works with zuul_console running
-  vars:
-    zuul_console_disabled: true
   hosts: node
   tasks:
     - name: Run quiet command
-      command: echo 'This command should not stream'
+      command: echo 'This command should not stream 2'
+
+- name: Enable zuul console
+  hosts: node
+  tasks:
+    - set_fact:
+        zuul_console_disabled: false
 
 - name: Run some commands to show that logging works
   hosts: node

playbooks/zuul-stream/functional.yaml

@@ -94,6 +94,12 @@
         egrep 'zuul_log_id": "skip"' job-output-success-19885.json
         egrep 'zuul_log_id": "skip"' job-output-success-19887.json
 
+    - name: Validate streaming connections new console
+      command: python3 src/opendev.org/zuul/zuul/playbooks/zuul-stream/validate.py console-job-output-success-19887.txt job-output-success-19887.txt job-output-success-19887.json
+
+    - name: Validate streaming connections old console
+      command: python3 src/opendev.org/zuul/zuul/playbooks/zuul-stream/validate.py console-job-output-success-19885.txt job-output-success-19885.txt job-output-success-19885.json
+
     # failure case
 
     - name: Run ansible playbook that should fail

playbooks/zuul-stream/post.yaml

@@ -19,6 +19,7 @@
         - logging.json
         - ansible.cfg
         - console-job-output-success-19887.txt
+        - console-job-output-success-19885.txt
         - job-output-success-19887.txt
         - job-output-success-19887.json
         - job-output-success-19885.txt

playbooks/zuul-stream/validate.py

@@ -0,0 +1,58 @@
+# Copyright 2025 Acme Gating, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import sys
+
+
+def main():
+    console_path = sys.argv[1]
+    output_txt_path = sys.argv[2]
+    output_json_path = sys.argv[3]
+    with open(output_json_path) as f:
+        output_json = json.loads(f.read())
+    with open(output_txt_path) as f:
+        output_txt = f.read()
+    with open(console_path) as f:
+        console = f.read()
+
+    # Get a list of all the task ids where we skippod log live
+    # streaming (but we get one for each host, so use a set to dedup).
+    skipped = set()
+
+    for playbook in output_json:
+        for play in playbook['plays']:
+            for task in play['tasks']:
+                for host in task['hosts'].values():
+                    logid = host.get('zuul_log_id')
+                    if logid == 'skip':
+                        skipped.add(task['task']['id'])
+                        # The only tasks that should do this are:
+                        assert "command should not stream" in host['stdout']
+
+    # We should have skipped 2 tasks
+    assert len(skipped) == 2
+    for skip in skipped:
+        # Verify that we did not attempt a connection
+        assert f"Starting to log {skip}" not in console
+
+    # We should have output due to copying from the result dict
+    assert "This command should not stream 1" in output_txt
+    assert "This command should not stream 2" in output_txt
+
+
+if __name__ == "__main__":
+    main()

zuul/ansible/base/callback/zuul_stream.py

@@ -487,6 +487,7 @@ class CallbackModule(default.CallbackModule):
                 if (ip in ('localhost', '127.0.0.1')):
                     # Don't try to stream from localhost
                     continue
+                # Note: this checks hostvars, but not task vars.
                 if boolean(play_vars[host].get(
                         'zuul_console_disabled', False)):
                     # The user has told us not to even try
@@ -597,6 +598,7 @@ class CallbackModule(default.CallbackModule):
             # We don't have streaming for localhost, raw, or disabled
             # consoles, so get standard out after the fact.
             play_vars = self._play._variable_manager._hostvars
+            # Note: this checks hostvars, but not task vars.
             is_disabled = boolean(play_vars[result._host.get_name()].get(
                 'zuul_console_disabled', False))
             if (is_localhost or