Collect journald logs

It is helpful to have journald logs when something goes wrong with
systemd.

Change-Id: I48765caf6917c2167da6052bcf90070b113d1f99
Signed-off-by: Paul Belanger <pabelanger@redhat.com>

tests/collect-logs.yaml

@@ -2,14 +2,43 @@
   tasks:
     - name: Ensure logs directory exists
       file:
-        path: "{{ zuul.executor.log_root }}/logs"
+        path: "{{ item }}"
         state: directory
+      with_items:
+        - "{{ zuul.executor.log_root }}/logs"
+        - "{{ zuul.executor.log_root }}/logs/gear01/var/log"
+        - "{{ zuul.executor.log_root }}/logs/nb01/var/log"
+        - "{{ zuul.executor.log_root }}/logs/nl01/var/log"
+        - "{{ zuul.executor.log_root }}/logs/ze01/var/log"
+        - "{{ zuul.executor.log_root }}/logs/zm01/var/log"
+        - "{{ zuul.executor.log_root }}/logs/zs01/var/log"
+        - "{{ zuul.executor.log_root }}/logs/zw01/var/log"
 
 - hosts: gear01
   tasks:
+    - name: Ensure journald logs directory exists
+      file:
+        path: "~/logs/{{ inventory_hostname }}/journal"
+        state: directory
+
+    - name: Collect journald logs
+      become: yes
+      shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      args:
+        creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      with_items:
+        - gear
+
+    - name: Collect journald log files
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
+        mode: pull
+        src: "~/logs/{{ inventory_hostname }}/journal"
+        verify_host: true
+
     - name: Collect gear log files
       synchronize:
-        dest: "{{ zuul.executor.log_root }}/logs/gear01"
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
         mode: pull
         rsync_opts:
           - "--relative"
@@ -21,9 +50,29 @@
 
 - hosts: nb01
   tasks:
+    - name: Ensure journald logs directory exists
+      file:
+        path: "~/logs/{{ inventory_hostname }}/journal"
+        state: directory
+
+    - name: Collect journald logs
+      become: yes
+      shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      args:
+        creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      with_items:
+        - nodepool-builder
+
+    - name: Collect journald log files
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
+        mode: pull
+        src: "~/logs/{{ inventory_hostname }}/journal"
+        verify_host: true
+
     - name: Collect nodepool log files
       synchronize:
-        dest: "{{ zuul.executor.log_root }}/logs/nb01"
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
         mode: pull
         rsync_opts:
           - "--relative"
@@ -38,9 +87,29 @@
 
 - hosts: nl01
   tasks:
+    - name: Ensure journald logs directory exists
+      file:
+        path: "~/logs/{{ inventory_hostname }}/journal"
+        state: directory
+
+    - name: Collect journald logs
+      become: yes
+      shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      args:
+        creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      with_items:
+        - nodepool-launcher
+
+    - name: Collect journald log files
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
+        mode: pull
+        src: "~/logs/{{ inventory_hostname }}/journal"
+        verify_host: true
+
     - name: Collect nl01 log files
       synchronize:
-        dest: "{{ zuul.executor.log_root }}/logs/nl01"
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
         mode: pull
         rsync_opts:
           - "--relative"
@@ -54,9 +123,29 @@
 
 - hosts: ze01
   tasks:
+    - name: Ensure journald logs directory exists
+      file:
+        path: "~/logs/{{ inventory_hostname }}/journal"
+        state: directory
+
+    - name: Collect journald logs
+      become: yes
+      shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      args:
+        creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      with_items:
+        - zuul-executor
+
+    - name: Collect journald log files
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
+        mode: pull
+        src: "~/logs/{{ inventory_hostname }}/journal"
+        verify_host: true
+
     - name: Collect ze01 log files
       synchronize:
-        dest: "{{ zuul.executor.log_root }}/logs/ze01"
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
         mode: pull
         rsync_opts:
           - "--relative"
@@ -71,9 +160,29 @@
 
 - hosts: zm01
   tasks:
+    - name: Ensure journald logs directory exists
+      file:
+        path: "~/logs/{{ inventory_hostname }}/journal"
+        state: directory
+
+    - name: Collect journald logs
+      become: yes
+      shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      args:
+        creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      with_items:
+        - zuul-merger
+
+    - name: Collect journald log files
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
+        mode: pull
+        src: "~/logs/{{ inventory_hostname }}/journal"
+        verify_host: true
+
     - name: Collect zm01 log files
       synchronize:
-        dest: "{{ zuul.executor.log_root }}/logs/zm01"
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
         mode: pull
         rsync_opts:
           - "--relative"
@@ -88,9 +197,29 @@
 
 - hosts: zs01
   tasks:
+    - name: Ensure journald logs directory exists
+      file:
+        path: "~/logs/{{ inventory_hostname }}/journal"
+        state: directory
+
+    - name: Collect journald logs
+      become: yes
+      shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      args:
+        creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      with_items:
+        - zuul-scheduler
+
+    - name: Collect journald log files
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
+        mode: pull
+        src: "~/logs/{{ inventory_hostname }}/journal"
+        verify_host: true
+
     - name: Collect zs01 log files
       synchronize:
-        dest: "{{ zuul.executor.log_root }}/logs/zs01"
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
         mode: pull
         rsync_opts:
           - "--relative"
@@ -105,9 +234,29 @@
 
 - hosts: zw01
   tasks:
+    - name: Ensure journald logs directory exists
+      file:
+        path: "~/logs/{{ inventory_hostname }}/journal"
+        state: directory
+
+    - name: Collect journald logs
+      become: yes
+      shell: "journalctl -u {{ item }}.service > /home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      args:
+        creates: "/home/{{ ansible_user }}/logs/{{ inventory_hostname }}/journal/{{ item }}.service.log"
+      with_items:
+        - zuul-web
+
+    - name: Collect journald log files
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}/var/log"
+        mode: pull
+        src: "~/logs/{{ inventory_hostname }}/journal"
+        verify_host: true
+
     - name: Collect zw01 log files
       synchronize:
-        dest: "{{ zuul.executor.log_root }}/logs/zw01"
+        dest: "{{ zuul.executor.log_root }}/logs/{{ inventory_hostname }}"
         mode: pull
         rsync_opts:
           - "--relative"