Add log artifacts for upgrade steps

Context: https://trello.com/c/vJwmd6SD/771-indexing-upgrade-logs-in-logstash
         https://trello.com/c/TU3JEtmg/560-ensure-upgrade-logging

- Per feedback have added timestamper_cmd so these
  logs can be indexed by logstash.

- for clarity have added a distinct variable for update template

- for clarity have added ffu_ prefix to FFU variant of UC upgrade template

- Per feedback removed the logging from bash jinja template
  as it's inconsistent with logging norms in TQ/TQE

I suggest that we also rename undercloud_upgrade.sh.j2 to reflect
that it's used in both upgrade and update workflows, or we clearly
disambiguate and fully denormalize by duplicating the template.  In order
to limit scope/risk IMHO this should be done in a discreet commit.

Change-Id: I184c5ebd28ae11a93a854df5dc2cafd02a5e1b6c
(cherry picked from commit baff4a2522)

defaults/main.yml

@@ -43,6 +43,10 @@ tripleo_ci: false
 undercloud_rc: "{{ working_dir }}/stackrc"
 overcloud_rc: "{{ working_dir }}/{{ overcloud_stack_name }}rc"
 
+# required for logstash indexing
+timestamper_cmd: >-
+    | awk '{ print strftime("%Y-%m-%d %H:%M:%S |"), $0; fflush(); }'
+
 # launch workload before update/upgrade
 workload_launch: false
 external_network_name: "public"
@@ -51,7 +55,9 @@ workload_memory: "512"
 workload_user: "cirros"
 workload_launch_post_composable_upgrade: false
 
+# note that both UC upgrade and update use the same template
 undercloud_upgrade_template: undercloud_upgrade.sh.j2
+undercloud_update_template:  undercloud_upgrade.sh.j2
 overcloud_upgrade_prepare_template: overcloud_upgrade_prepare.sh.j2
 overcloud_upgrade_run_template: overcloud_upgrade_run.sh.j2
 overcloud_upgrade_converge_template: overcloud_upgrade_converge.sh.j2

tasks/update/create-undercloud-update-scripts.yaml

@@ -1,9 +1,7 @@
 ---
 - name: create undercloud update script
-  vars:
-    log_prefix: "update"
   template:
-    src: "{{ undercloud_upgrade_template }}"
+    src:  "{{ undercloud_update_template }}"
     dest: "{{ undercloud_update_script }}"
     mode: 0775
     force: true

tasks/upgrade/create-undercloud-upgrade-scripts.yaml

@@ -1,9 +1,7 @@
 ---
 - name: create undercloud upgrade script
-  vars:
-    log_prefix: "upgrade"
   template:
-    src: "{{ undercloud_upgrade_template }}"
+    src:  "{{ undercloud_upgrade_template }}"
     dest: "{{ undercloud_upgrade_script }}"
     mode: 0775
     force: true

tasks/upgrade/main.yml

@@ -6,11 +6,19 @@
 
 - block:
     - name: apply pre undercloud upgrade workarounds
-      command: "{{working_dir}}/pre_undercloud_upgrade_workarounds.sh"
+      shell: |
+        set -o pipefail
+        pre_undercloud_upgrade_workarounds.sh 2>&1 {{ timestamper_cmd }} > pre_undercloud_upgrade_workarounds.log
+      args:
+        chdir: "{{working_dir}}"
       when: upgrade_workarounds
 
     - name: upgrade undercloud
-      shell: "bash {{ undercloud_upgrade_script }} &> {{ working_dir }}/undercloud_upgrade.log"
+      shell: |
+        set -o pipefail
+        bash {{ undercloud_upgrade_script }} 2>&1 {{ timestamper_cmd }} > undercloud_upgrade.log
+      args:
+        chdir: "{{working_dir}}"
       tags: undercloud_upgrade
 
     - name: validate undercloud upgrade
@@ -18,7 +26,11 @@
       tags: undercloud_upgrade_validate
 
     - name: apply post undercloud upgrade workarounds
-      command: "{{working_dir}}/post_undercloud_upgrade_workarounds.sh"
+      shell: |
+        set -o pipefail
+        post_undercloud_upgrade_workarounds.sh 2>&1 {{ timestamper_cmd }} > post_undercloud_upgrade_workarounds.log
+      args:
+        chdir: "{{working_dir}}"
       when: upgrade_workarounds
   when: undercloud_upgrade|bool
 
@@ -29,7 +41,11 @@
 
 - block:
     - name: apply pre overcloud upgrade prepare workarounds
-      command: "{{working_dir}}/pre_overcloud_upgrade_prepare_workarounds.sh"
+      shell: |
+        set -o pipefail
+        pre_overcloud_upgrade_prepare_workarounds.sh 2>&1 {{ timestamper_cmd }} > pre_overcloud_upgrade_prepare_workarounds.log
+      args:
+        chdir: "{{working_dir}}"
       when: upgrade_workarounds
       tags: overcloud_upgrade_prepare
 
@@ -37,7 +53,11 @@
       tags: overcloud_upgrade_prepare
 
     - name: apply post overcloud upgrade prepare workarounds
-      command: "{{working_dir}}/post_overcloud_upgrade_prepare_workarounds.sh"
+      shell: |
+        set -o pipefail
+        post_overcloud_upgrade_prepare_workarounds.sh 2>&1 {{ timestamper_cmd }} > post_overcloud_upgrade_prepare_workarounds.log
+      args:
+        chdir: "{{working_dir}}"
       when: upgrade_workarounds
       tags: overcloud_upgrade_prepare
 
@@ -48,19 +68,31 @@
       become: true
 
     - name: apply pre overcloud upgrade workarounds
-      command: "{{working_dir}}/pre_overcloud_upgrade_workarounds.sh"
+      shell: |
+        set -o pipefail
+        pre_overcloud_upgrade_workarounds.sh 2>&1 {{ timestamper_cmd }} > pre_overcloud_upgrade_workarounds.log
+      args:
+        chdir: "{{working_dir}}"
       when: upgrade_workarounds
       tags: overcloud_upgrade_run
 
     - name: launch workload
-      command: "{{ workload_launch_script }}"
+      shell: |
+        set -o pipefail
+        {{ workload_launch_script }} 2>&1 {{ timestamper_cmd }} > workload_launch.log
+      args:
+        chdir: "{{working_dir}}"
       when: workload_launch_post_composable_upgrade|bool
 
     - include: overcloud_upgrade_run.yml
       tags: overcloud_upgrade_run
 
     - name: apply post overcloud upgrade workarounds
-      command: "{{working_dir}}/post_overcloud_upgrade_workarounds.sh"
+      shell: |
+        set -o pipefail
+        post_overcloud_upgrade_workarounds.sh 2>&1 {{ timestamper_cmd }} > post_overcloud_upgrade_workarounds.log
+      args:
+        chdir: "{{working_dir}}"
       when: upgrade_workarounds
       tags: overcloud_upgrade_run
 
@@ -69,7 +101,11 @@
       when: ceph_osd_enabled|bool
 
     - name: apply pre overcloud upgrade converge workarounds
-      command: "{{working_dir}}/pre_overcloud_upgrade_converge_workarounds.sh"
+      shell: |
+        set -o pipefail
+        pre_overcloud_converge_workarounds.sh 2>&1 {{ timestamper_cmd }} > pre_overcloud_converge_workarounds.log
+      args:
+        chdir: "{{working_dir}}"
       when: upgrade_workarounds
       tags: overcloud_upgrade_converge
 
@@ -77,7 +113,11 @@
       tags: overcloud_upgrade_converge
 
     - name: apply post overcloud upgrade converge workarounds
-      command: "{{working_dir}}/post_overcloud_upgrade_converge_workarounds.sh"
+      shell: |
+        set -o pipefail
+        post_overcloud_converge_workarounds.sh 2>&1 {{ timestamper_cmd }} > post_overcloud_converge_workarounds.log
+      args:
+        chdir: "{{working_dir}}"
       when: upgrade_workarounds
       tags: overcloud_upgrade_converge
 

templates/undercloud_upgrade.sh.j2

@@ -4,4 +4,4 @@
 set -euo pipefail
 
 sudo yum update -y python-tripleoclient
-openstack undercloud upgrade &> undercloud_{{ log_prefix }}.log
+openstack undercloud upgrade