Switch to new log collecting process

Change-Id: Iedce72d782484d478eebe6b7e3c7fd1fd7cb56c9 Signed-off-by: Paul Belanger <pabelanger@redhat.com>
2018-08-20 20:22:42 -04:00 · 2018-08-20 20:22:42 -04:00 · b6b5235734
parent 153e4ce713
commit b6b5235734
3 changed files with 81 additions and 101 deletions
--- a/tests/collect-logs.yaml
+++ b/tests/collect-logs.yaml
@ -1,109 +1,101 @@
- hosts: statsd01
+- hosts: all,!bastion
  tasks:
    - name: Ensure journald logs directory exists
      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
+        path: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal"
        state: directory

+- hosts: statsd01
+  tasks:
    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      shell: "sudo journalctl -u {{ item }}.service | tee {{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+        creates: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      with_items:
        - statsd

    - name: Collect statsd log files
+      become: yes
      synchronize:
-        dest: "~/logs/{{ inventory_hostname }}"
-        mode: pull
+        dest: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}"
        rsync_opts:
          - "--relative"
+          - "--chown={{ ansible_user_id }}:{{ ansible_user_id }}"
        src: "{{ item }}"
        verify_host: true
+      delegate_to: "{{ inventory_hostname }}"
      with_items:
        - /etc/statsd
        - /var/log/statsd

 - hosts: gear01
  tasks:
-    - name: Ensure journald logs directory exists
-      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
-        state: directory
-
    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      shell: "sudo journalctl -u {{ item }}.service | tee {{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+        creates: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      with_items:
        - gear

    - name: Collect gear log files
+      become: yes
      synchronize:
-        dest: "~/logs/{{ inventory_hostname }}"
-        mode: pull
-        recursive: false
+        dest: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}"
        rsync_opts:
          - "--relative"
+          - "--chown={{ ansible_user_id }}:{{ ansible_user_id }}"
        src: "{{ item }}"
        verify_host: true
+      delegate_to: "{{ inventory_hostname }}"
      with_items:
        - /etc/gear/logging.conf
-        - /var/log/gear/*.log
+        - /var/log/gear

 - hosts: nb01
  tasks:
-    - name: Ensure journald logs directory exists
-      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
-        state: directory
-
    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      shell: "sudo journalctl -u {{ item }}.service | tee {{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+        creates: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      with_items:
        - nodepool-builder

-    - name: Collect nodepool log files
+    - name: Collect nodepool-builder log files
+      become: yes
      synchronize:
-        dest: "~/logs/{{ inventory_hostname }}"
-        mode: pull
-        recursive: false
+        dest: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}"
        rsync_opts:
          - "--relative"
+          - "--chown={{ ansible_user_id }}:{{ ansible_user_id }}"
        src: "{{ item }}"
        verify_host: true
+      delegate_to: "{{ inventory_hostname }}"
      with_items:
        - /etc/nodepool/builder-logging.conf
        - /etc/nodepool/nodepool.yaml
-        - /var/log/nodepool/builds/*.log
+        - /var/log/nodepool/builds
        - /var/log/nodepool/builder-debug.log
        - /var/log/nodepool/nodepool-builder.log

 - hosts: nl01
  tasks:
-    - name: Ensure journald logs directory exists
-      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
-        state: directory
-
    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      shell: "sudo journalctl -u {{ item }}.service | tee {{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+        creates: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      with_items:
        - nodepool-launcher

-    - name: Collect nl01 log files
+    - name: Collect nodepool-launcher log files
+      become: yes
      synchronize:
-        dest: "~/logs/{{ inventory_hostname }}"
-        mode: pull
-        recursive: false
+        dest: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}"
        rsync_opts:
          - "--relative"
+          - "--chown={{ ansible_user_id }}:{{ ansible_user_id }}"
        src: "{{ item }}"
        verify_host: true
+      delegate_to: "{{ inventory_hostname }}"
      with_items:
        - /etc/nodepool/launcher-logging.conf
        - /etc/nodepool/nodepool.yaml
@ -112,27 +104,23 @@

 - hosts: zs01
  tasks:
-    - name: Ensure journald logs directory exists
-      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
-        state: directory
-
    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      shell: "sudo journalctl -u {{ item }}.service | tee {{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+        creates: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      with_items:
        - zuul-scheduler

-    - name: Collect zs01 log files
+    - name: Collect zuul-scheduler log files
+      become: yes
      synchronize:
-        dest: "~/logs/{{ inventory_hostname }}"
-        mode: pull
-        recursive: false
+        dest: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}"
        rsync_opts:
          - "--relative"
+          - "--chown={{ ansible_user_id }}:{{ ansible_user_id }}"
        src: "{{ item }}"
        verify_host: true
+      delegate_to: "{{ inventory_hostname }}"
      with_items:
        - /etc/zuul/main.yaml
        - /etc/zuul/scheduler-logging.conf
@ -142,27 +130,23 @@

 - hosts: ze01
  tasks:
-    - name: Ensure journald logs directory exists
-      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
-        state: directory
-
    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      shell: "sudo journalctl -u {{ item }}.service | tee {{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+        creates: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      with_items:
        - zuul-executor

-    - name: Collect ze01 log files
+    - name: Collect zuul-executor log files
+      become: yes
      synchronize:
-        dest: "~/logs/{{ inventory_hostname }}"
-        mode: pull
-        recursive: false
+        dest: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}"
        rsync_opts:
          - "--relative"
+          - "--chown={{ ansible_user_id }}:{{ ansible_user_id }}"
        src: "{{ item }}"
        verify_host: true
+      delegate_to: "{{ inventory_hostname }}"
      with_items:
        - /etc/zuul/executor-logging.conf
        - /etc/zuul/zuul.conf
@ -171,27 +155,23 @@

 - hosts: zf01
  tasks:
-    - name: Ensure journald logs directory exists
-      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
-        state: directory
-
    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      shell: "sudo journalctl -u {{ item }}.service | tee {{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+        creates: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      with_items:
        - zuul-fingergw

-    - name: Collect zf01 log files
+    - name: Collect zuul-fingergw log files
+      become: yes
      synchronize:
-        dest: "~/logs/{{ inventory_hostname}}"
-        mode: pull
-        recursive: false
+        dest: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}"
        rsync_opts:
          - "--relative"
+          - "--chown={{ ansible_user_id }}:{{ ansible_user_id }}"
        src: "{{ item }}"
        verify_host: true
+      delegate_to: "{{ inventory_hostname }}"
      with_items:
        - /etc/zuul/fingergw-logging.conf
        - /etc/zuul/zuul.conf
@ -200,27 +180,23 @@

 - hosts: zm01
  tasks:
-    - name: Ensure journald logs directory exists
-      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
-        state: directory
-
    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      shell: "sudo journalctl -u {{ item }}.service | tee {{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+        creates: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      with_items:
        - zuul-merger

-    - name: Collect zm01 log files
+    - name: Collect zuul-merger log files
+      become: yes
      synchronize:
-        dest: "~/logs/{{ inventory_hostname }}"
-        mode: pull
-        recursive: false
+        dest: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}"
        rsync_opts:
          - "--relative"
+          - "--chown={{ ansible_user_id }}:{{ ansible_user_id }}"
        src: "{{ item }}"
        verify_host: true
+      delegate_to: "{{ inventory_hostname }}"
      with_items:
        - /etc/zuul/merger-logging.conf
        - /etc/zuul/zuul.conf
@ -229,27 +205,23 @@

 - hosts: zw01
  tasks:
-    - name: Ensure journald logs directory exists
-      file:
-        path: "~/logs/{{ inventory_hostname }}/var/log/journal"
-        state: directory
-
    - name: Collect journald logs
-      shell: "sudo journalctl -u {{ item }}.service | tee ~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+      shell: "sudo journalctl -u {{ item }}.service | tee {{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      args:
-        creates: "~/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
+        creates: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}/var/log/journal/{{ item }}.service.log"
      with_items:
        - zuul-web

-    - name: Collect zw01 log files
+    - name: Collect zuul-web log files
+      become: yes
      synchronize:
-        dest: "~/logs/{{ inventory_hostname }}"
-        mode: pull
-        recursive: false
+        dest: "{{ zuul_output_dir }}/logs/logs/{{ inventory_hostname }}"
        rsync_opts:
          - "--relative"
+          - "--chown={{ ansible_user_id }}:{{ ansible_user_id }}"
        src: "{{ item }}"
        verify_host: true
+      delegate_to: "{{ inventory_hostname }}"
      with_items:
        - /etc/zuul/web-logging.conf
        - /etc/zuul/zuul.conf
--- a/tests/playbooks/post.yaml
+++ b/tests/playbooks/post.yaml
@ -12,20 +12,22 @@
        - name: Run ansible-playbook for collect-logs.yaml
          args:
            chdir: "{{ windmill_src_dir }}"
-          shell: tox -evenv -- ansible-playbook -i inventory/testing/hosts tests/collect-logs.yaml
+          shell: "tox -evenv -- ansible-playbook -i inventory/testing/hosts -ezuul_output_dir={{ zuul_output_dir }} tests/collect-logs.yaml"

      always:
        - name: Ensure ara-report directory exists
          file:
-            path: ~/logs/ara-report
+            path: "{{ zuul_output_dir }}/logs/logs/ara-report"
            state: directory

        - name: Copy ARA database to ara-report directory
-          shell: cp ~/.ara/ansible.sqlite ~/logs/ara-report
+          shell: "cp ~/.ara/ansible.sqlite {{ zuul_output_dir }}/logs/logs/ara-report"

-        - name: Collect log files
+        # TODO: Migrate to fetch-zuul-logs when
+        # https://review.openstack.org/#/c/583346/ is merged.
+        - name: Collect log output
          synchronize:
-            dest: "{{ zuul.executor.log_root }}"
+            dest: "{{ zuul.executor.log_root }}/"
            mode: pull
-            src: ~/logs
+            src: "{{ zuul_output_dir }}/logs/"
            verify_host: true
--- a/tests/playbooks/pre.yaml
+++ b/tests/playbooks/pre.yaml
@ -1,5 +1,11 @@
 - hosts: all
  tasks:
+    # TODO(pabelanger): Remove once this lands in our base job in
+    # project-config.
+    - name: Execute ensure-output-dirs role
+      include_role:
+        name: ensure-output-dirs
+
    - name: Disable extra wheels mirror
      become: yes
      lineinfile: