HTTP test during update/upgrade/ffwd.

To mimic real life environment during update/upgrade/ffwd
 we'll expose a 'HTTP-live' test.
This test asserts that web server running in overcloud is
 reachable via its FIP and that content served is accessible.

To access WEB server(s) running in overcloud appropriate ports
 80 and 443 have to be added to the security group(s).

Initial work done in https://review.openstack.org/#/c/547220/

Co-Authored-By: Marius Cornea <mcornea@redhat.com>

Change-Id: I818949e0e498d5cd8da4640bda2be99aafd1aa76

defaults/main.yml

@@ -148,6 +148,9 @@ l3_agent_connectivity_check_start_template: l3_agent_start_ping.sh.j2
 l3_agent_connectivity_check_stop_template: l3_agent_stop_ping.sh.j2
 l3_agent_failover_check: false
 
+# enable web load test
+fip_http_check: false
+
 # migrate instances between compute nodes during upgrade
 compute_evacuate: false
 

tasks/common/create_http_test_scripts.yml

@@ -0,0 +1,9 @@
+---
+- name: Create FIP HTTP check scripts
+  template:
+    src: "{{ item }}"
+    dest: "{{working_dir}}/{{ item.split('.') | first }}.sh"
+    mode: 0775
+  with_items:
+      - 'fip_http_check_start.sh.j2'
+      - 'fip_http_check_stop.sh.j2'

tasks/fast-forward-upgrade/create-prepare-scripts.yaml

@@ -79,3 +79,6 @@
 
 - name: create nova actions check script
   include: ../common/create_nova_actions_check_script.yml
+
+- name: import create HTTP test scripts
+  import_tasks: ../common/create_http_test_scripts.yml

tasks/update/create-overcloud-update-scripts.yaml

@@ -73,3 +73,6 @@
 
 - name: create nova actions check script
   include: ../common/create_nova_actions_check_script.yml
+
+- name: import create HTTP test scripts
+  import_tasks: ../common/create_http_test_scripts.yml

tasks/upgrade/create-overcloud-upgrade-scripts.yaml

@@ -94,3 +94,6 @@
 
 - name: create nova actions check script
   include: ../common/create_nova_actions_check_script.yml
+
+- name: import create HTTP test scripts
+  import_tasks: ../common/create_http_test_scripts.yml

templates/fip_http_check_start.sh.j2

@@ -0,0 +1,10 @@
+#!/bin/env bash
+
+FIP=$(openstack floating ip list -f json | jq -r -c '.[] | select(.Port) | .["Floating IP Address"]' | head -1)
+
+while true; do
+    curl --location --silent --show-error --max-time 5 \
+        --write-out "HTTP Code: %{http_code}\nTotal Time: %{time_total} sec\nEffective URL: %{url_effective}\n\n" \
+        http://$FIP/ 2>/dev/null >> {{ working_dir }}/http_response.log
+    sleep 1
+done

templates/fip_http_check_stop.sh.j2

@@ -0,0 +1,9 @@
+#!/bin/env bash
+
+RESULT_CODE=$(grep 'HTTP Code' {{ working_dir }}/http_response.log | sort | uniq)
+
+#TODO: assert other conditions/thresholds for pass/fail criteria
+if [[ "$RESULT_CODE" != "HTTP Code: 200" ]]; then
+    echo 'There are non-200 HTTP calls'
+    exit 1
+fi

templates/overcloud_upgrade_converge.sh.j2

@@ -17,6 +17,13 @@ if [[ -e {{ working_dir }}/l3_agent_failover_pre.sh ]]; then
 fi
 {% endif %}
 
+{% if fip_http_check|bool %}
+source {{ overcloud_rc }}
+kill -9 $( lsof -t {{ working_dir }}/fip_http_check_start.sh )
+bash {{ working_dir }}/fip_http_check_start.sh &
+{% endif %}
+
+
 source {{ undercloud_rc }}
 echo "[$(date)] Running {% if ffu_overcloud_upgrade %} ffwd-upgrade {% else %} major {% endif %}upgrade converge step"
 openstack overcloud {% if ffu_overcloud_upgrade %} ffwd-upgrade {% else %} upgrade {% endif %}converge \
@@ -76,3 +83,9 @@ if [[ -e {{ working_dir }}/l3_agent_failover_post.sh ]]; then
     bash {{ working_dir }}/l3_agent_failover_post.sh
 fi
 {% endif %}
+
+{% if fip_http_check|bool %}
+source {{ overcloud_rc }}
+kill -9 $( lsof -t {{ working_dir }}/fip_http_check_start.sh )
+bash {{ working_dir }}/fip_http_check_stop.sh
+{% endif %}

templates/overcloud_upgrade_run.sh.j2

@@ -22,6 +22,12 @@ if [[ -e {{ working_dir }}/l3_agent_failover_pre.sh ]]; then
 fi
 {% endif %}
 
+{% if fip_http_check|bool %}
+source {{ overcloud_rc }}
+kill -9 $( lsof -t {{ working_dir }}/fip_http_check_start.sh )
+bash {{ working_dir }}/fip_http_check_start.sh &
+{% endif %}
+
 source {{ undercloud_rc }}
 
 echo "[$(date)] Runing major upgrade {{ playbook }} playbook for {{ item }} role"
@@ -51,6 +57,13 @@ if [[ -e {{ working_dir }}/l3_agent_failover_post.sh ]]; then
 fi
 {% endif %}
 
+{% if fip_http_check|bool %}
+source {{ overcloud_rc }}
+kill -9 $( lsof -t {{ working_dir }}/fip_http_check_start.sh )
+bash {{ working_dir }}/fip_http_check_stop.sh
+{% endif %}
+
+
     {%- endfor %}
 {% else %}
 
@@ -68,6 +81,12 @@ if [[ -e {{ working_dir }}/l3_agent_failover_pre.sh ]]; then
 fi
 {% endif %}
 
+{% if fip_http_check|bool %}
+source {{ overcloud_rc }}
+kill -9 $( lsof -t {{ working_dir }}/fip_http_check_start.sh )
+bash {{ working_dir }}/fip_http_check_start.sh &
+{% endif %}
+
 source {{ undercloud_rc }}
 
 echo "[$(date)] Runing major upgrade for {{ item }} role"
@@ -97,4 +116,10 @@ if [[ -e {{ working_dir }}/l3_agent_failover_post.sh ]]; then
     bash {{ working_dir }}/l3_agent_failover_post.sh
 fi
 {% endif %}
+
+{% if fip_http_check|bool %}
+source {{ overcloud_rc }}
+kill -9 $( lsof -t {{ working_dir }}/fip_http_check_start.sh )
+bash {{ working_dir }}/fip_http_check_stop.sh
+{% endif %}
 {% endif %}

templates/workload_launch.sh.j2

@@ -65,6 +65,8 @@ if [ $? -ne 0 ]; then
     openstack security group create ${SECGROUP_NAME}
     openstack security group rule create --proto icmp ${SECGROUP_NAME}
     openstack security group rule create --proto tcp --dst-port 22 ${SECGROUP_NAME}
+    openstack security group rule create --proto tcp --dst-port 80 ${SECGROUP_NAME}
+    openstack security group rule create --proto tcp --dst-port 443 ${SECGROUP_NAME}
 fi
 
 ## create instance