Add timestamps to update log files

In order for logstash indexing to function, logfiles need timestamps

This commit "updates the update logs" to be consistent with other
logs collected by CI. Note that the timestamper_cmd has an embedded "|"

Related Change (upgrade logging):

- https://git.openstack.org/cgit/openstack/tripleo-upgrade/commit/?id=baff4a25229b1904e960e426208eca76c96c3ab4

Change-Id: I47b6ff912aa045c266b903fa148b068be653cae2

tasks/update/ceph_update_run.yml

@@ -1,7 +1,9 @@
 ---
 - name: run Ceph minor update
   shell: |
-    {{ working_dir }}/ceph-update-run.sh &> {{ working_dir }}/ceph-update-run.log
+    set -o pipefail
+    {{ working_dir }}/ceph-update-run.sh 2>&1 {{ timestamper_cmd }} > \
+      {{ working_dir }}/ceph-update-run.log
   ignore_errors: true
   register: ceph_update_status
 

tasks/update/main.yml

@@ -7,18 +7,25 @@
 - block:
     - name: undercloud pre-update workarounds
       shell: |
+          set -o pipefail
           source {{ undercloud_rc }}
-          bash {{ pre_undercloud_update_workarounds_script }} &> {{ working_dir }}/uc_pre_update_workarounds.log
+          bash {{ pre_undercloud_update_workarounds_script }} 2>&1 {{ timestamper_cmd }} > \
+            {{ working_dir }}/uc_pre_update_workarounds.log
       when: updates_workarounds|bool
 
     - name: update undercloud
-      shell: "bash {{ undercloud_update_script }} &> {{ working_dir }}/undercloud_update.log"
+      shell: |
+          set -o pipefail
+          bash {{ undercloud_update_script }} 2>&1 {{ timestamper_cmd }} > \
+            {{ working_dir }}/undercloud_update.log
       tags: undercloud_update
 
     - name: undercloud post-update workarounds
       shell: |
+          set -o pipefail
           source {{ undercloud_rc }}
-          bash {{ post_undercloud_update_workarounds_script }} &> {{ working_dir }}/uc_pre_update_workarounds.log
+          bash {{ post_undercloud_update_workarounds_script }} 2>&1 {{ timestamper_cmd }} > \
+            {{ working_dir }}/uc_pre_update_workarounds.log
       when: updates_workarounds|bool
 
     - name: validate undercloud update
@@ -35,8 +42,10 @@
 - block:
     - name: overcloud pre-update prepare workarounds
       shell: |
+          set -o pipefail
           source {{ undercloud_rc }}
-          bash {{ pre_overcloud_update_prepare_workarounds_script }} &> {{ working_dir }}/pre_overcloud_update_prepare_workarounds.log
+          bash {{ pre_overcloud_update_prepare_workarounds_script }} 2>&1 {{ timestamper_cmd }} > \
+            {{ working_dir }}/pre_overcloud_update_prepare_workarounds.log
       when: updates_workarounds|bool
 
     - include: overcloud_update_prepare.yml
@@ -46,8 +55,10 @@
 
     - name: overcloud post-update prepare workarounds
       shell: |
+          set -o pipefail
           source {{ undercloud_rc }}
-          bash {{ post_overcloud_update_prepare_workarounds_script }} &> {{ working_dir }}/post_overcloud_update_prepare_workarounds.log
+          bash {{ post_overcloud_update_prepare_workarounds_script }} 2>&1 {{ timestamper_cmd }} > \
+            {{ working_dir }}/post_overcloud_update_prepare_workarounds.log
       when: updates_workarounds|bool
 
     - name: Ensure ansible-pacemaker module is present in CI.
@@ -58,8 +69,10 @@
 
     - name: overcloud pre-update run workarounds
       shell: |
+          set -o pipefail
           source {{ undercloud_rc }}
-          bash {{ pre_overcloud_update_run_workarounds_script }} &> {{ working_dir }}/pre_overcloud_update_run_workarounds.log
+          bash {{ pre_overcloud_update_run_workarounds_script }} 2>&1 {{ timestamper_cmd }} > \
+            {{ working_dir }}/pre_overcloud_update_run_workarounds.log
       when: updates_workarounds|bool
 
     - include: overcloud_update_run.yml
@@ -68,8 +81,10 @@
 
     - name: overcloud post-update run workarounds
       shell: |
+          set -o pipefail
           source {{ undercloud_rc }}
-          bash {{ post_overcloud_update_run_workarounds_script }} &> {{ working_dir }}/post_overcloud_update_run_workarounds.log
+          bash {{ post_overcloud_update_run_workarounds_script }} 2>&1 {{ timestamper_cmd }} > \
+            {{ working_dir }}/post_overcloud_update_run_workarounds.log
       when: updates_workarounds|bool
 
     - name: update Ceph
@@ -91,8 +106,10 @@
 
     - name: validate overcloud docker images/containers
       shell: |
+          set -o pipefail
           source {{ undercloud_rc }}
-          bash {{ overcloud_validate_images_script }} &> {{ working_dir}}/validate_oc_images_containers.log
+          bash {{ overcloud_validate_images_script }} 2>&1 {{ timestamper_cmd }} > \
+            {{ working_dir}}/validate_oc_images_containers.log
       tags:
           - overcloud_update
           - overcloud_images_validate

tasks/update/overcloud_update_converge.yml

@@ -2,7 +2,10 @@
 - include: ../common/l3_agent_connectivity_check_start_script.yml
 
 - name: run minor update converge step
-  shell: "bash {{ overcloud_update_converge_script }} &> {{ working_dir }}/overcloud_update_converge.log"
+  shell: |
+        set -o pipefail
+        bash {{ overcloud_update_converge_script }} 2>&1 {{ timestamper_cmd }} > \
+          {{ working_dir }}/overcloud_update_converge.log
   register: overcloud_converge_update
   ignore_errors: yes
 
@@ -11,7 +14,8 @@
       shell: |
         set -o pipefail
         source {{ undercloud_rc }}
-        openstack stack failures list --long {{ overcloud_stack_name }} | tee {{ working_dir}}/overcloud_composable_failed_update_converge.log
+        openstack stack failures list --long {{ overcloud_stack_name }} {{ timestamper_cmd }} | \
+          tee {{ working_dir}}/overcloud_composable_failed_update_converge.log
 
     - name: was the overcloud update converge successful.
       fail: msg="Overcloud update converge step failed... :("

tasks/update/overcloud_update_prepare.yml

@@ -1,8 +1,10 @@
 ---
 - name: setup HEAT outputs via update prepare
   shell: |
+    set -o pipefail
     source {{ undercloud_rc }}
-    bash {{ overcloud_update_prepare_script }} &> {{ working_dir }}/overcloud_update_prepare.log
+    bash {{ overcloud_update_prepare_script }} 2>&1 {{ timestamper_cmd }} > \
+      {{ working_dir }}/overcloud_update_prepare.log
   register: overcloud_update_prepare
   ignore_errors: yes
 
@@ -11,13 +13,15 @@
       shell: |
         set -o pipefail
         source {{ undercloud_rc }}
-        openstack stack failures list --long {{ overcloud_stack_name }} | tee {{ working_dir}}/overcloud_failed_update_prepare.log
+        openstack stack failures list --long {{ overcloud_stack_name }} {{ timestamper_cmd }} | \
+          tee {{ working_dir}}/overcloud_failed_update_prepare.log
 
     - name: print resource list
       shell: |
         set -o pipefail
         source {{ undercloud_rc }}
-        openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} | tee {{ working_dir}}/overcloud_failed_prepare_resources.log
+        openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} {{ timestamper_cmd }} | \
+          tee {{ working_dir}}/overcloud_failed_prepare_resources.log
 
     - name: was the overcloud update prepare successful.
       fail: msg="Overcloud minor update preparation step failed..."

tasks/update/overcloud_update_run.yml

@@ -2,7 +2,10 @@
 - include: ../common/l3_agent_connectivity_check_start_script.yml
 
 - name: run overcloud minor update in each of the roles/hostgroups
-  shell: "bash {{ overcloud_update_run_script_base }}-{{ item }}.sh &> {{ working_dir}}/overcloud_update_run_{{ item }}.log"
+  shell: |
+        set -o pipefail
+        bash {{ overcloud_update_run_script_base }}-{{ item }}.sh 2>&1 {{ timestamper_cmd }} > \
+          {{ working_dir}}/overcloud_update_run_{{ item }}.log
   with_items:
       - "{{ oc_roles|default('all') }}"
   register: overcloud_update_nodes