Add post upgrade Nova operation check

Add check that validates that Nova operations can be performed
against an instance already created before the upgrade/update.
This check reboots the instance via Nova API call and validates
the reboot by checking the uptime via SSH.

Change-Id: I6ca773c03ef31b5b2fad72ac2671bbc209a47af2

defaults/main.yml

@@ -49,6 +49,7 @@ workload_launch: false
 external_network_name: "public"
 workload_image_url: "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img"
 workload_memory: "512"
+workload_user: "cirros"
 workload_launch_post_composable_upgrade: false
 
 undercloud_upgrade_template: undercloud_upgrade.sh.j2
@@ -199,3 +200,6 @@ overcloud_upgrade_playbooks: ['upgrade_steps_playbook.yaml', 'deploy_steps_playb
 
 # Ansible playbooks executed during "overcloud update run"
 overcloud_update_playbooks: ['update_steps_playbook.yaml', 'deploy_steps_playbook.yaml' ]
+
+# Post upgrade/update nova actions tests
+nova_actions_check: false

infrared_plugin/main.yml

@@ -24,6 +24,10 @@
          workload_memory: "{{ install.upgrade.workloadmemory }}"
       when: install.upgrade.workload
 
+    - name: Set upgrade workload user
+      set_fact:
+         workload_user: "{{ install.upgrade.workloaduser }}"
+
     - name: Set upgrade workload post_composable_upgrade launch
       set_fact:
          workload_launch_post_composable_upgrade: true
@@ -175,5 +179,10 @@
       set_fact:
          l3_agent_failover_check: true
       when: install.upgrade.l3agent.failover.check
+
+    - name: Set upgrade nova_actions_check
+      set_fact:
+         nova_actions_check: true
+      when: install.upgrade.nova.actions.check
   roles:
     - tripleo-upgrade

infrared_plugin/plugin.spec

@@ -55,6 +55,11 @@ subparsers:
                       help: |
                           Memory assigned to the instance spawned before upgrade
                       default: 512
+                  upgrade-workloaduser:
+                      type: Value
+                      help: |
+                          User used for conecting to workload instance via SSH
+                      default: cirros
                   upgrade-compute-evacuate:
                       type: Bool
                       help: |
@@ -104,6 +109,12 @@ subparsers:
                           Check l3 agent does not failover during upgrade.
                           Existing neutron router is required.
                       default: false
+                  upgrade-nova-actions-check:
+                      type: Bool
+                      help: |
+                          Check Nova actions can be performed to an already existing
+                          instance post upgrade.
+                      default: false
             - title: TripleO Update
               options:
                   overcloud-update:

tasks/common/create_nova_actions_check_script.yml

@@ -0,0 +1,6 @@
+- name: create post nova actions test
+  template:
+    src: "nova_actions_check.sh.j2"
+    dest: "{{ working_dir }}/nova_actions_check.sh"
+    mode: 0775
+  when: nova_actions_check|bool

tasks/common/nova_actions_check.yml

@@ -0,0 +1,6 @@
+---
+- name: run post nova actions test
+  shell: |
+    source {{ overcloud_rc }}
+    {{ working_dir }}/nova_actions_check.sh | tee {{ working_dir }}/nova_actions_check.log
+  when: nova_actions_check|bool

tasks/fast-forward-upgrade/main.yml

@@ -119,4 +119,6 @@
         tags: ffu_upgrade_ceph
       when: ceph_ansible|succeeded
 
+    - include: ../common/nova_actions_check.yml
+
   when: ffu_overcloud_upgrade|bool

tasks/update/main.yml

@@ -75,4 +75,6 @@
           - overcloud_update
           - overcloud_images_validate
 
+    - include: ../common/nova_actions_check.yml
+
   when: overcloud_update|bool

tasks/upgrade/main.yml

@@ -76,5 +76,8 @@
       include: controller_post_upgrade.yml
       tags: controller_post_upgrade
       when: controller_upgrade_post|bool
+
+    - include: ../common/nova_actions_check.yml
+
   when: overcloud_upgrade|bool
 

templates/nova_actions_check.sh.j2

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -eux
+
+SERVER_ID=$(openstack server list -f json | jq -r -c '.[0] | .ID')
+SERVER_FIP=$(openstack server show -f json ${SERVER_ID} | jq -r -c '.addresses' | grep -oP ',.*' | sed s/,\ //)
+SERVER_SSH_USER={{ workload_user }}
+
+## reboot
+echo "Rebooting ${SERVER_ID} nova instance"
+timeout 2m openstack server reboot --wait ${SERVER_ID}
+rc=$?
+if [ $rc -eq 124 ]; then
+    echo "Rebooting ${SERVER_ID} nova instance timed out"
+    exit 1
+elif [ $rc -ne 0 ]; then
+    echo "Rebooting ${SERVER_ID} nova instance failed with unknown reason"
+    exit 1
+fi
+
+## wait for instance to be reachable over the floating IP and check uptime
+timeout_seconds=240
+elapsed_seconds=0
+while true; do
+    echo "Waiting for ${SERVER_ID} to be reachable over its floating IP"
+    INSTANCE_UPTIME=$(ssh -q -o StrictHostKeyChecking=no -o ConnectTimeout=1 ${SERVER_SSH_USER}@${SERVER_FIP} 'uptime' | awk {'print $3$4'} | sed s/,//)
+    if [ "$INSTANCE_UPTIME" == '0min' ]; then
+        echo "SUCCESS: ${SERVER_ID} nova instance successfully rebooted and is reachable via its floating IP"
+        break
+    fi
+    sleep 3
+    elapsed_seconds=$(expr $elapsed_seconds + 3)
+    if [ $elapsed_seconds -ge $timeout_seconds ]; then
+        echo "FAILURE: ${SERVER_ID} nova instance did not reboot or it's not reachable via its floating IP after reboot"
+        exit 1
+    fi
+done