zuul-stream : Test against a Python 2.7 container

Change Ief366c092e05fb88351782f6d9cd280bfae96237 intoduced a bug in
the streaming daemons because it was using Python 3.6 features.  The
streaming console needs to work on all Ansible managed nodes, which
includes back to Python 2.7 nodes (while Ansible supports that).

This introduces a regression test by building about the smallest
Python 2.7 container that can be managed by Ansbile.  We start this
container and modify the test inventory to include it, then run the
stream tests against it.

The existing testing runs against the "new" console but also tests
against the console OpenDev's Zuul starts to ensure
backwards-compatability.  Since this container wasn't started by Zuul
it doesn't have this, so that testing is skipped for this node.

It might be good to abstract all testing of the console daemons into
separate containers for each Ansible supported managed-node Python
version -- it's a bit more work than I want to take on right now.
This should ensure the lower-bound though and prevent regressions for
older platforms.

Change-Id: Ia78ad9e3ec51bc47bf68c9ff38c0fcd16ba2e728

playbooks/zuul-stream/2.7-container.yaml

@@ -0,0 +1,21 @@
+- name: Install docker
+  include_role:
+    name: ensure-docker
+
+- name: Build 2.7 container environment
+  shell: |
+    pushd {{ ansible_user_dir }}/src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/
+    cat ~/.ssh/id_rsa.pub > authorized_keys
+    docker build -f Dockerfile.py27 -t zuul_python27 .
+  args:
+    executable: /bin/bash
+
+- name: Run 2.7 container
+  shell: |
+    docker run -d -p 2022:22 -p 19887:19887 zuul_python27
+    docker ps
+
+- name: Accept host keys
+  shell: |
+    ssh-keyscan -p 2022 localhost >> ~/.ssh/known_hosts
+    ssh-keyscan -p 2022 127.0.0.1 >> ~/.ssh/known_hosts

playbooks/zuul-stream/create-inventory.yaml

@@ -0,0 +1,38 @@
+- name: Copy inventory
+  copy:
+    src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml"
+    dest: "{{ ansible_user_dir }}/inventory.yaml"
+
+- name: Slurp inventory
+  slurp:
+    path: "{{ ansible_user_dir }}/inventory.yaml"
+  register: _inventory_yaml
+
+- name: Extract inventory
+  set_fact:
+    _new_inventory: "{{ _inventory_yaml['content'] | b64decode | from_yaml }}"
+
+- name: Setup new facts
+  set_fact:
+    _docker_inventory:
+      all:
+        children:
+          node:
+            hosts:
+              node3: null
+        hosts:
+          node3:
+            ansible_connection: ssh
+            ansible_host: 127.0.0.1
+            ansible_port: 2022
+            ansible_user: root
+            ansible_python_interpreter: /usr/local/bin/python2.7
+
+- name: Merge all facts
+  set_fact:
+    _new_inventory: '{{ _new_inventory | combine(_docker_inventory, recursive=True) }}'
+
+- name: Write out inventory
+  copy:
+    content: '{{ _new_inventory | to_nice_yaml }}'
+    dest: '{{ ansible_user_dir }}/inventory.yaml'

playbooks/zuul-stream/fixtures/Dockerfile.py27

@@ -0,0 +1,24 @@
+FROM python:2.7.18-buster AS buster-2.7-ssh
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update \
+  && apt-get install -y dumb-init openssh-server \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /var/run/sshd && chmod 0755 /var/run/sshd
+
+# This may or not be required to allow logins by preventing pam_loginuid
+# trying to write out audit level things that may not work in a container
+RUN sed -ri 's/session(\s+)required(\s+)pam_loginuid.so/session\1optional\2pam_loginuid.so/' /etc/pam.d/sshd
+
+RUN ssh-keygen -A -v
+
+RUN ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519
+
+COPY authorized_keys /root/.ssh/authorized_keys
+RUN chmod 0600 /root/.ssh/authorized_keys
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["/usr/sbin/sshd", "-D", "-o", "ListenAddress=0.0.0.0" ]

playbooks/zuul-stream/functional.yaml

@@ -19,6 +19,7 @@
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
         ZUUL_JOBDIR: "{{ ansible_user_dir}}"
         PYTHONPATH: "{{ python_path }}"
+        ZUUL_CONSOLE_STREAM_LOCALHOST: 1
       register: _success_output
 
     - name: Save raw output to file
@@ -48,10 +49,13 @@
     # NOTE(ianw) 2022-07 : we deliberatly have this second step to run
     # against the console setup by the infrastructure executor in the
     # job pre playbooks as a backwards compatability sanity check.
+    # The py27 container job (node3) is not running an existing
+    # console streamer, so that will not output anything -- limit this
+    # out.
     - name: Run ansible that should succeed against extant console
       command: >
         /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook
-        -e "new_console=false"
+        -e "new_console=false" --limit="node1,node2"
         src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml
       environment:
         ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
@@ -76,6 +80,8 @@
         - { node: 'node2', filename: 'job-output-success-19887.txt' }
         - { node: 'node1', filename: 'job-output-success-19885.txt' }
         - { node: 'node2', filename: 'job-output-success-19885.txt' }
+        # node3 only listen on 19887
+        - { node: 'node3', filename: 'job-output-success-19887.txt' }
 
     # failure case
 
@@ -100,8 +106,10 @@
       shell: |
         egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output-failure.txt
         egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output-failure.txt
+        egrep "^.+\| node3 \| Exception: Test module failure exception fail-task" job-output-failure.txt
 
     - name: Validate output - failure item loop with exception
       shell: |
         egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
         egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output-failure.txt
+        egrep "^.+\| node3 \| Exception: Test module failure exception fail-loop" job-output-failure.txt

playbooks/zuul-stream/pre.yaml

@@ -9,6 +9,12 @@
 
   post_tasks:
 
+    - name: Setup 2.7 container environment
+      include_tasks: 2.7-container.yaml
+
+    - name: Setup inventory
+      include_tasks: create-inventory.yaml
+
     - name: Install pip
       shell: |+
         python3 -m pip install --upgrade pip setuptools wheel
@@ -36,11 +42,6 @@
         # venvs) and the installation fails due to conflicts.
         SETUPTOOLS_USE_DISTUTILS: stdlib
 
-    - name: Copy inventory
-      copy:
-        src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml"
-        dest: "{{ ansible_user_dir }}/inventory.yaml"
-
     - name: Copy ansible.cfg
       template:
         src: templates/ansible.cfg.j2

zuul/ansible/base/callback/zuul_stream.py

@@ -49,6 +49,8 @@ from zuul.ansible import logconfig
 
 LOG_STREAM_PORT = int(os.environ.get("ZUUL_CONSOLE_PORT", 19885))
 LOG_STREAM_VERSION = 0
+#
+LOG_STREAM_LOCALHOST = int(os.environ.get("ZUUL_CONSOLE_STREAM_LOCALHOST", 0))
 
 
 def zuul_filter_result(result):
@@ -319,13 +321,15 @@ class CallbackModule(default.CallbackModule):
             hosts = self._get_task_hosts(task)
             for host, inventory_hostname in hosts:
                 port = LOG_STREAM_PORT
-                if host in ('localhost', '127.0.0.1'):
+                if (host in ('localhost', '127.0.0.1') and
+                    not LOG_STREAM_LOCALHOST):
                     # Don't try to stream from localhost
                     continue
                 ip = play_vars[host].get(
                     'ansible_host', play_vars[host].get(
                         'ansible_inventory_host'))
-                if ip in ('localhost', '127.0.0.1'):
+                if (ip in ('localhost', '127.0.0.1') and
+                    not LOG_STREAM_LOCALHOST):
                     # Don't try to stream from localhost
                     continue
                 if play_vars[host].get('ansible_connection') in ('winrm',):