Add timestamps to logs of upgrade tasks

As continuation of I184c5ebd28ae11a93a854df5dc2cafd02a5e1b6c
we add timestamps to logs

Change-Id: I1524efe39efc09b31ec69430832e4bb964172943

tasks/common/nova_actions_check.yml

@@ -3,5 +3,6 @@
   shell: |
     set -o pipefail
     source {{ overcloud_rc }}
-    {{ working_dir }}/nova_actions_check.sh | tee {{ working_dir }}/nova_actions_check.log
+    {{ working_dir }}/nova_actions_check.sh 2>&1 {{ timestamper_cmd }} | \
+      tee {{ working_dir }}/nova_actions_check.log
   when: nova_actions_check|bool

tasks/fast-forward-upgrade/ffu_update_stack_outputs.yaml

@@ -2,7 +2,8 @@
 - name: update stack outputs
   shell: |
     source {{ undercloud_rc }}
-    {{ working_dir }}/ffu_update_stack_outputs.sh &> {{ working_dir }}/ffu_update_stack_outputs.log
+    {{ working_dir }}/ffu_update_stack_outputs.sh 2>&1 {{ timestamper_cmd }} > \
+      {{ working_dir }}/ffu_update_stack_outputs.log
   register: ffu_stack_output
   ignore_errors: yes
 
@@ -10,14 +11,16 @@
   shell: |
     set -o pipefail
     source {{ undercloud_rc }}
-    openstack stack failures list --long {{ overcloud_stack_name }} | tee {{ working_dir}}/ffu_update_stack_outputs_failed.log
+    openstack stack failures list --long {{ overcloud_stack_name }} 2>&1 {{ timestamper_cmd }} | \
+      tee {{ working_dir}}/ffu_update_stack_outputs_failed.log
   when: ffu_stack_output|failed
 
 - name: print resource list
   shell: |
     set -o pipefail
     source {{ undercloud_rc }}
-    openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} | tee {{ working_dir}}/ffu_update_stack_outputs_failed_resources.log
+    openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} 2>&1 {{ timestamper_cmd }} | \
+      tee {{ working_dir}}/ffu_update_stack_outputs_failed_resources.log
   when: ffu_stack_output|failed
 
 - name: was the ffu update stack outputs successful.

tasks/fast-forward-upgrade/ffu_upgrade_ceph.yaml

@@ -1,7 +1,8 @@
 - name: upgrade ceph post ffu
   shell: |
     source {{ undercloud_rc }}
-    {{ working_dir }}/ffu_upgrade_ceph_script.sh &> {{ working_dir }}/ffu_upgrade_ceph.log
+    {{ working_dir }}/ffu_upgrade_ceph_script.sh 2>&1 {{ timestamper_cmd }} > \
+    {{ working_dir }}/ffu_upgrade_ceph.log
   register: ffu_upgrade_ceph
   ignore_errors: yes
 
@@ -10,13 +11,15 @@
       shell: |
         set -o pipefail
         source {{ undercloud_rc }}
-        openstack stack failures list --long {{ overcloud_stack_name }} | tee {{ working_dir}}/ffu_upgrade_ceph_failed.log
+        openstack stack failures list --long {{ overcloud_stack_name }} 2>&1 {{ timestamper_cmd }} | \
+        tee {{ working_dir}}/ffu_upgrade_ceph_failed.log
 
     - name: print resource list
       shell: |
         set -o pipefail
         source {{ undercloud_rc }}
-        openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} | tee {{ working_dir}}/ffu_upgrade_ceph_failed_resources.log
+        openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} \
+        2>&1 {{ timestamper_cmd }} | tee {{ working_dir}}/ffu_upgrade_ceph_failed_resources.log
 
     - name: was the ffu update ceph successful.
       fail: msg="FFU upgrade ceph step failed... :("

tasks/fast-forward-upgrade/ffu_upgrade_converge.yaml

@@ -2,7 +2,8 @@
 - name: run ffu converge step
   shell: |
     source {{ undercloud_rc }}
-    {{ working_dir }}/ffu_upgrade_converge_script.sh &> {{ working_dir }}/ffu_upgrade_converge.log
+    {{ working_dir }}/ffu_upgrade_converge_script.sh 2>&1 {{ timestamper_cmd }} > \
+      {{ working_dir }}/ffu_upgrade_converge.log
   register: ffu_converge
   ignore_errors: yes
 
@@ -10,14 +11,16 @@
   shell: |
     set -o pipefail
     source {{ undercloud_rc }}
-    openstack stack failures list --long {{ overcloud_stack_name }} | tee {{ working_dir}}/ffu_upgrade_converge_failed.log
+    openstack stack failures list --long {{ overcloud_stack_name }} 2>&1 {{ timestamper_cmd }} | \
+    tee {{ working_dir}}/ffu_upgrade_converge_failed.log
   when: ffu_converge|failed
 
 - name: print resource list
   shell: |
     set -o pipefail
     source {{ undercloud_rc }}
-    openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} | tee {{ working_dir}}/ffu_upgrade_converge_failed_resources.log
+    openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} \
+    2>&1 {{ timestamper_cmd }} | tee {{ working_dir}}/ffu_upgrade_converge_failed_resources.log
   when: ffu_converge|failed
 
 - name: was the ffu converge successful.

tasks/upgrade/ceph_upgrade_run.yml

@@ -1,7 +1,8 @@
 ---
 - name: run Ceph upgrade
   shell: |
-    {{ working_dir }}/ceph-upgrade-run.sh &> {{ working_dir }}/ceph-upgrade-run.log
+    {{ working_dir }}/ceph-upgrade-run.sh 2>&1 {{ timestamper_cmd }} > \
+      {{ working_dir }}/ceph-upgrade-run.log
   ignore_errors: true
   register: ceph_upg_status
 

tasks/upgrade/overcloud_upgrade_converge.yml

@@ -2,7 +2,8 @@
 - include: ../common/l3_agent_connectivity_check_start_script.yml
 
 - name: run major upgrade converge step
-  shell: "bash {{ overcloud_upgrade_converge_script }} &> {{ working_dir }}/overcloud_upgrade_converge.log"
+  shell: "bash {{ overcloud_upgrade_converge_script }} 2>&1 {{ timestamper_cmd }} > \
+    {{ working_dir }}/overcloud_upgrade_converge.log"
   register: overcloud_converge_upgrade
   ignore_errors: yes
 
@@ -11,7 +12,8 @@
       shell: |
         set -o pipefail
         source {{ undercloud_rc }}
-        openstack stack failures list --long {{ overcloud_stack_name }} | tee {{ working_dir}}/overcloud_composable_failed_upgrade_converge.log
+        openstack stack failures list --long {{ overcloud_stack_name }} 2>&1 {{ timestamper_cmd }} | \
+          tee {{ working_dir}}/overcloud_composable_failed_upgrade_converge.log
 
     - name: was the overcloud upgrade converge successful.
       fail: msg="Overcloud upgrade converge step failed... :("

tasks/upgrade/overcloud_upgrade_prepare.yml

@@ -19,7 +19,8 @@
     force: true
 
 - name: setup HEAT outputs via upgrade prepare
-  shell: "bash {{ overcloud_upgrade_prepare_script }} &> {{ working_dir }}/overcloud_upgrade_prepare.log"
+  shell: "bash {{ overcloud_upgrade_prepare_script }} 2>&1 {{ timestamper_cmd }} > \
+    {{ working_dir }}/overcloud_upgrade_prepare.log"
   register: overcloud_upgrade_prepare
   ignore_errors: yes
 
@@ -28,13 +29,15 @@
       shell: |
         set -o pipefail
         source {{ undercloud_rc }}
-        openstack stack failures list --long {{ overcloud_stack_name }} | tee {{ working_dir}}/overcloud_failed_upgrade_prepare.log
+        openstack stack failures list --long {{ overcloud_stack_name }} 2>&1 {{ timestamper_cmd }} | \
+        tee {{ working_dir}}/overcloud_failed_upgrade_prepare.log
 
     - name: print resource list
       shell: |
         set -o pipefail
         source {{ undercloud_rc }}
-        openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} | tee {{ working_dir}}/overcloud_failed_prepare_resources.log
+        openstack stack resource list --filter status=FAILED --nested-depth 5 {{ overcloud_stack_name }} \
+        2>&1 {{ timestamper_cmd }} | tee {{ working_dir}}/overcloud_failed_prepare_resources.log
 
     - name: was the overcloud upgrade preparation successful.
       fail: msg="Overcloud upgrade preparation step failed... :("

tasks/upgrade/overcloud_upgrade_run.yml

@@ -1,6 +1,7 @@
 ---
 - name: run overcloud major upgrade in each of the roles/hostgroups
-  shell: "bash {{ overcloud_upgrade_run_script_base }}-{{ item }}.sh &> {{ working_dir }}/overcloud_upgrade_run_{{ item }}.log"
+  shell: "bash {{ overcloud_upgrade_run_script_base }}-{{ item }}.sh \
+    2>&1 {{ timestamper_cmd }} > {{ working_dir }}/overcloud_upgrade_run_{{ item }}.log"
   with_items:
       - "{{ oc_roles|default('all') }}"
   register: overcloud_upgrade_nodes