Update tests/collect-logs.yaml for new scheduler order

When we updated I235b4d7aa9f96aae0f75d8370a410b8fe0793d61 we didn't
change the order of our log collection.

We also add some error handling around tests/collect-logs.yaml to
ensure we always generate ara-reports and rsync what ever logs we
gather.

Change-Id: Ib2bbd95be19b2963dd7eac44827112c100fdfe0a
Signed-off-by: Paul Belanger <pabelanger@redhat.com>

tests/collect-logs.yaml

@@ -110,6 +110,36 @@
         - /var/log/nodepool/launcher-debug.log
         - /var/log/nodepool/nodepool-launcher.log
 
+- hosts: zs01
+  tasks:
+    - name: Ensure journald logs directory exists
+      file:
+        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
+        state: directory
+
+    - name: Collect journald logs
+      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      args:
+        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      with_items:
+        - zuul-scheduler
+
+    - name: Collect zs01 log files
+      synchronize:
+        dest: "~/logs/{{ inventory_hostname }}"
+        mode: pull
+        recursive: false
+        rsync_opts:
+          - "--relative"
+        src: "{{ item }}"
+        verify_host: true
+      with_items:
+        - /etc/zuul/main.yaml
+        - /etc/zuul/scheduler-logging.conf
+        - /etc/zuul/zuul.conf
+        - /var/log/zuul/scheduler-debug.log
+        - /var/log/zuul/scheduler.log
+
 - hosts: ze01
   tasks:
     - name: Ensure journald logs directory exists
@@ -197,36 +227,6 @@
         - /var/log/zuul/merger-debug.log
         - /var/log/zuul/merger.log
 
-- hosts: zs01
-  tasks:
-    - name: Ensure journald logs directory exists
-      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
-        state: directory
-
-    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
-      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
-      with_items:
-        - zuul-scheduler
-
-    - name: Collect zs01 log files
-      synchronize:
-        dest: "~/logs/{{ inventory_hostname }}"
-        mode: pull
-        recursive: false
-        rsync_opts:
-          - "--relative"
-        src: "{{ item }}"
-        verify_host: true
-      with_items:
-        - /etc/zuul/main.yaml
-        - /etc/zuul/scheduler-logging.conf
-        - /etc/zuul/zuul.conf
-        - /var/log/zuul/scheduler-debug.log
-        - /var/log/zuul/scheduler.log
-
 - hosts: zw01
   tasks:
     - name: Ensure journald logs directory exists

tests/playbooks/post.yaml

@@ -1,21 +1,24 @@
 - hosts: all
   tasks:
-    - name: Run ansible-playbook for collect-logs.yaml
-      args:
-        chdir: "{{ windmill_src_dir }}"
-      shell: tox -evenv -- ansible-playbook -i inventory/testing/hosts tests/collect-logs.yaml
+    - name: Collect logs from hosts
+      block:
+        - name: Run ansible-playbook for collect-logs.yaml
+          args:
+            chdir: "{{ windmill_src_dir }}"
+          shell: tox -evenv -- ansible-playbook -i inventory/testing/hosts tests/collect-logs.yaml
 
-    - name: Ensure ara-report directory exists
-      file:
-        path: ~/logs/ara-report
-        state: directory
+      always:
+        - name: Ensure ara-report directory exists
+          file:
+            path: ~/logs/ara-report
+            state: directory
 
-    - name: Copy ARA database to ara-report directory
-      shell: cp ~/.ara/ansible.sqlite ~/logs/ara-report
+        - name: Copy ARA database to ara-report directory
+          shell: cp ~/.ara/ansible.sqlite ~/logs/ara-report
 
-    - name: Collect log files
-      synchronize:
-        dest: "{{ zuul.executor.log_root }}"
-        mode: pull
-        src: ~/logs
-        verify_host: true
+        - name: Collect log files
+          synchronize:
+            dest: "{{ zuul.executor.log_root }}"
+            mode: pull
+            src: ~/logs
+            verify_host: true