Add integration test for zuul_stream

Actually run a playbook against a second node with the plugins enabled,
then fetch the output so we can look at it.

Takes advantage of the fact that zuul_console will already be running on
the second node.

Change-Id: I75bcbc46ffbbb49f0689584fbdb1c62ac2e91737

.zuul.yaml

@@ -1,3 +1,21 @@
+- nodeset:
+    name: zuul-two-node
+    nodes:
+      - name: controller
+        label: ubuntu-xenial
+      - name: node
+        label: ubuntu-xenial
+
+- job:
+    name: zuul-stream-functional
+    parent: multinode
+    nodes: zuul-two-node
+    pre-run: playbooks/zuul-stream/pre
+    run: playbooks/zuul-stream/functional
+    post-run: playbooks/zuul-stream/post
+    required-projects:
+      - openstack/ara
+
 - project:
     name: openstack-infra/zuul
     check:
@@ -7,6 +25,7 @@
             voting: false
         - tox-pep8
         - tox-py35
+        - zuul-stream-functional
     gate:
       jobs:
         - tox-docs

playbooks/zuul-stream/fixtures/test-stream.yaml

@@ -0,0 +1,21 @@
+- name: Run some commands to show that logging works
+  hosts: node
+  tasks:
+
+    - name: Run setup
+      setup:
+      register: setupvar
+
+    - name: Output debug for a var
+      debug:
+        var: setupvar
+
+    - name: Run a shell task
+      command: ip addr show
+
+    - name: Loop with items
+      command: "echo {{ item }}"
+      with_items:
+        - item1
+        - item2
+        - item3

playbooks/zuul-stream/functional.yaml

@@ -0,0 +1,14 @@
+- hosts: controller
+  tasks:
+
+    - name: Run ansible
+      command: ansible-playbook -vvv src/git.openstack.org/openstack-infra/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml
+      environment:
+        ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
+        ARA_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json"
+
+    - name: Generate ARA html
+      command: ara generate html ara-output
+
+    - name: Compress ARA html
+      command: gzip --recursive --best ara-output

playbooks/zuul-stream/post.yaml

@@ -0,0 +1,26 @@
+- hosts: controller
+  tasks:
+
+    - set_fact:
+        output_dir: "{{ zuul.executor.log_root }}/stream-files"
+
+    - name: Make log subdir
+      file:
+        path: "{{ output_dir }}"
+        state: directory
+      delegate_to: localhost
+
+    - name: Rename job-output.txt
+      command: mv job-output.txt stream-job-output.txt
+
+    - name: Fetch files
+      synchronize:
+        src: "{{ ansible_user_dir }}/{{ item }}"
+        dest: "{{ output_dir }}"
+        mode: pull
+      with_items:
+        - logging.json
+        - ansible.cfg
+        - stream-job-output.txt
+        - job-output.json
+        - ara-output

playbooks/zuul-stream/pre.yaml

@@ -0,0 +1,30 @@
+- hosts: controller
+  roles:
+
+    - role: bindep
+      bindep_profile: test
+      bindep_dir: src/git.openstack.org/openstack-infra/zuul
+      bindep_command: /usr/bindep-env/bin/bindep
+
+    - role: bindep
+      bindep_dir: src/git.openstack.org/openstack/ara
+      bindep_command: /usr/bindep-env/bin/bindep
+
+  post_tasks:
+
+    - name: Install software
+      command: python3 -m pip install src/git.openstack.org/openstack-infra/zuul src/git.openstack.org/openstack/ara
+      become: yes
+
+    - name: Copy inventory
+      copy:
+        src: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml"
+        dest: "{{ ansible_user_dir }}/inventory.yaml"
+
+    - name: Copy ansible.cfg
+      template:
+        src: templates/ansible.cfg.j2
+        dest: "{{ ansible_user_dir }}/ansible.cfg"
+
+    - name: Generate logging config
+      command: python3 src/git.openstack.org/openstack-infra/zuul/zuul/ansible/logconfig.py

playbooks/zuul-stream/templates/ansible.cfg.j2

@@ -0,0 +1,11 @@
+[defaults]
+hostfile = {{ ansible_user_dir }}/inventory.yaml
+gathering = smart
+gather_subset = !all
+lookup_plugins = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/lookup
+action_plugins = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/action
+callback_plugins = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/callback:{{ ansible_user_dir }}/src/git.openstack.org/openstack/ara/ara/plugins/callbacks
+module_utils = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/module_utils
+stdout_callback = zuul_stream
+library = {{ ansible_user_dir }}/src/git.openstack.org/openstack-infra/zuul/zuul/ansible/library
+retry_files_enabled = False