From 0c0ec6e5802259bfc7da1b75a9e5d574c2227002 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 19 Jul 2022 13:46:53 +1000 Subject: [PATCH] zuul-stream: start new console for stream tests The zuul-stream-functional tests currently run Ansible against two hosts (node1 and node2) started by the "infrastructure" Zuul. However the tests are working with the zuul_console running on the default port 19885 -- which is the zuul_console started by the setup jobs at [1]. The result of this is that this test is only talking to a zuul_console instance started from the current executor's code, not the Zuul checkout uder test -- i.e. changes to zuul_console.py aren't tested by this change. This modifies the job playbook to have another step that can start its own zuul_console service on the two hosts running at another port (19887). This way we can test against the zuul_console code from the Zuul checkout. A new step is added to run a playbook overriding to this port. We retain the existing test (against the already running port 19885) as a backwards compatability test; although we don't exactly know what this is running (as it comes from OpenDev's production executors) we want changes to the console log/stream callback to be able to run against it, as it may represent what is running a static node. The test results are pulled apart a bit to be more explicit, logging and testing each run separately. [1] https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/prepare-workspace/tasks/main.yaml#L2 Change-Id: Ib11b77cfdc6c59d12807c6d9684c3e653ccad863 --- .../zuul-stream/fixtures/test-stream.yaml | 13 +++ playbooks/zuul-stream/functional.yaml | 86 ++++++++++--------- playbooks/zuul-stream/post.yaml | 11 +-- playbooks/zuul-stream/validate.yaml | 38 ++++++++ 4 files changed, 101 insertions(+), 47 deletions(-) create mode 100644 playbooks/zuul-stream/validate.yaml diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml index 05b75daf99..0c10304b2b 100644 --- a/playbooks/zuul-stream/fixtures/test-stream.yaml +++ b/playbooks/zuul-stream/fixtures/test-stream.yaml @@ -1,3 +1,16 @@ +- name: Start zuul stream daemon + hosts: node + tasks: + + # NOTE : when new_console is set, this playbook runs with + # ZUUL_CONSOLE_PORT=19887 so that we test with the zuul_console + # from the Zuul checkout, and not the one started by test + # infrastructure. + - name: Start zuul_console on non-default port + zuul_console: + port: 19887 + when: new_console | default(false) + - name: Run some commands to show that logging works hosts: node tasks: diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml index ee1643fbc2..3b79e7b40a 100644 --- a/playbooks/zuul-stream/functional.yaml +++ b/playbooks/zuul-stream/functional.yaml @@ -7,15 +7,50 @@ # the python version of the platform is changed. python_path: "/usr/local/lib/python3.10/dist-packages" - - name: Run ansible that should succeed + - name: Run ansible that should succeed against testing console command: > /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook + -e "new_console=true" + src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml + environment: + # Setup by test-stream.yaml so we start a new zuul_console + # from this checkout. + ZUUL_CONSOLE_PORT: 19887 + ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json" + ZUUL_JOBDIR: "{{ ansible_user_dir}}" + PYTHONPATH: "{{ python_path }}" + + - name: Save output + shell: | + mv job-output.txt job-output-success-19887.txt + mv job-output.json job-output-success-19887.json + + # NOTE(ianw) 2022-07 : we deliberatly have this second step to run + # against the console setup by the infrastructure executor in the + # job pre playbooks as a backwards compatability sanity check. + - name: Run ansible that should succeed against extant console + command: > + /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook + -e "new_console=false" src/opendev.org/zuul/zuul/playbooks/zuul-stream/fixtures/test-stream.yaml environment: ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json" ZUUL_JOBDIR: "{{ ansible_user_dir}}" PYTHONPATH: "{{ python_path }}" + - name: Save output + shell: | + mv job-output.txt job-output-success-19885.txt + mv job-output.json job-output-success-19885.json + + - name: Validate outputs + include_tasks: validate.yaml + loop: + - job-output-success-19887.txt + - job-output-success-19885.txt + + # failure case + - name: Run ansible playbook that should fail command: > /usr/lib/zuul/ansible/{{ zuul_ansible_version }}/bin/ansible-playbook @@ -23,55 +58,22 @@ register: failed_results failed_when: "failed_results.rc != 2" environment: + ZUUL_CONSOLE_PORT: 19887 ZUUL_JOB_LOG_CONFIG: "{{ ansible_user_dir}}/logging.json" ZUUL_JOBDIR: "{{ ansible_user_dir}}" PYTHONPATH: "{{ python_path }}" - - name: Validate output - setupvar + - name: Save output shell: | - egrep "^.*\| node1 \|\s+\"setupvar\": {" job-output.txt - egrep "^.*\| node2 \|\s+\"setupvar\": {" job-output.txt - - - name: Validate output - shell task - shell: | - egrep "^.*\| node1 \| 1: lo:" job-output.txt - egrep "^.*\| node2 \| 1: lo:" job-output.txt - - - name: Validate output - loop with items - shell: | - egrep "^.+\| node1 \| ok: Item: item1" job-output.txt - egrep "^.+\| node1 \| ok: Item: item2" job-output.txt - egrep "^.+\| node1 \| ok: Item: item3" job-output.txt - egrep "^.+\| node2 \| ok: Item: item1" job-output.txt - egrep "^.+\| node2 \| ok: Item: item2" job-output.txt - egrep "^.+\| node2 \| ok: Item: item3" job-output.txt - - - name: Validate output - loop with complex items - shell: | - egrep "^.+\| node1 \| ok: Item: Runtime" job-output.txt - egrep "^.+\| node2 \| ok: Item: Runtime" job-output.txt - - - name: Validate output - failed shell task - shell: | - egrep "^.+\| node1 \| Exception: Test module failure exception task" job-output.txt - egrep "^.+\| node2 \| Exception: Test module failure exception task" job-output.txt - - - name: Validate output - item loop with exception - shell: | - egrep "^.+\| node1 \| Exception: Test module failure exception loop" job-output.txt - egrep "^.+\| node2 \| Exception: Test module failure exception loop" job-output.txt + mv job-output.txt job-output-failure.txt + mv job-output.json job-output-failure.json - name: Validate output - failure shell task with exception shell: | - egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output.txt - egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output.txt + egrep "^.+\| node1 \| Exception: Test module failure exception fail-task" job-output-failure.txt + egrep "^.+\| node2 \| Exception: Test module failure exception fail-task" job-output-failure.txt - name: Validate output - failure item loop with exception shell: | - egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output.txt - egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output.txt - - - name: Validate output - binary data - shell: | - egrep "^.*\| node1 \| \\\\x80abc" job-output.txt - egrep "^.*\| node2 \| \\\\x80abc" job-output.txt + egrep "^.+\| node1 \| Exception: Test module failure exception fail-loop" job-output-failure.txt + egrep "^.+\| node2 \| Exception: Test module failure exception fail-loop" job-output-failure.txt diff --git a/playbooks/zuul-stream/post.yaml b/playbooks/zuul-stream/post.yaml index 2c717a82ea..a937718814 100644 --- a/playbooks/zuul-stream/post.yaml +++ b/playbooks/zuul-stream/post.yaml @@ -10,9 +10,6 @@ state: directory delegate_to: localhost - - name: Rename job-output.txt - command: mv job-output.txt stream-job-output.txt - - name: Fetch files synchronize: src: "{{ ansible_user_dir }}/{{ item }}" @@ -21,5 +18,9 @@ with_items: - logging.json - ansible.cfg - - stream-job-output.txt - - job-output.json + - job-output-success-19887.txt + - job-output-success-19887.json + - job-output-success-19885.txt + - job-output-success-19885.json + - job-output-failure.txt + - job-output-failure.json diff --git a/playbooks/zuul-stream/validate.yaml b/playbooks/zuul-stream/validate.yaml new file mode 100644 index 0000000000..73ccd873af --- /dev/null +++ b/playbooks/zuul-stream/validate.yaml @@ -0,0 +1,38 @@ +- name: Validate output - setupvar + shell: | + egrep "^.*\| node1 \|\s+\"setupvar\": {" {{ item }} + egrep "^.*\| node2 \|\s+\"setupvar\": {" {{ item }} + +- name: Validate output - shell task + shell: | + egrep "^.*\| node1 \| 1: lo:" {{ item }} + egrep "^.*\| node2 \| 1: lo:" {{ item }} + +- name: Validate output - loop with items + shell: | + egrep "^.+\| node1 \| ok: Item: item1" {{ item }} + egrep "^.+\| node1 \| ok: Item: item2" {{ item }} + egrep "^.+\| node1 \| ok: Item: item3" {{ item }} + egrep "^.+\| node2 \| ok: Item: item1" {{ item }} + egrep "^.+\| node2 \| ok: Item: item2" {{ item }} + egrep "^.+\| node2 \| ok: Item: item3" {{ item }} + +- name: Validate output - loop with complex items + shell: | + egrep "^.+\| node1 \| ok: Item: Runtime" {{ item }} + egrep "^.+\| node2 \| ok: Item: Runtime" {{ item }} + +- name: Validate output - failed shell task + shell: | + egrep "^.+\| node1 \| Exception: Test module failure exception task" {{ item }} + egrep "^.+\| node2 \| Exception: Test module failure exception task" {{ item }} + +- name: Validate output - item loop with exception + shell: | + egrep "^.+\| node1 \| Exception: Test module failure exception loop" {{ item }} + egrep "^.+\| node2 \| Exception: Test module failure exception loop" {{ item }} + +- name: Validate output - binary data + shell: | + egrep "^.*\| node1 \| \\\\x80abc" {{ item }} + egrep "^.*\| node2 \| \\\\x80abc" {{ item }}