Refactor CI testing scripts

This change is intended to ease the addition of new test scenarios, by
splitting the existing setup_gate.sh script up into several smaller
scripts. We may wish to further break these scripts up in future,
depending on the requirements of new test jobs.

Change-Id: Ifa442ac79a54f749c56d20f5f7383c3852d95268

tests/check-failure.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+check_failure() {
+    # All docker container's status are created, restarting, running, removing,
+    # paused, exited and dead. Containers without running status are treated as
+    # failure. removing is added in docker 1.13, just ignore it now.
+    failed_containers=$(sudo docker ps -a --format "{{.Names}}" \
+        --filter status=created \
+        --filter status=restarting \
+        --filter status=paused \
+        --filter status=exited \
+        --filter status=dead)
+
+    if [[ -n "$failed_containers" ]]; then
+        exit 1;
+    fi
+}
+
+
+check_failure

tests/deploy-bifrost.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function deploy_bifrost {
+    RAW_INVENTORY=/etc/kolla/inventory
+
+    # TODO(mgoddard): run prechecks.
+    # Deploy the bifrost container.
+    # TODO(mgoddard): add pull action when we have a local registry service in
+    # CI.
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost
+}
+
+
+deploy_bifrost

tests/deploy.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function deploy {
+    RAW_INVENTORY=/etc/kolla/inventory
+
+    # Create dummy interface for neutron
+    ansible -m shell -i ${RAW_INVENTORY} -b -a "ip l a fake_interface type dummy" all
+
+    #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
+    sudo chmod -R 777 /etc/kolla
+    # Actually do the deployment
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/deploy-prechecks
+    # TODO(jeffrey4l): add pull action when we have a local registry
+    # service in CI
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy
+}
+
+
+deploy

tests/reconfigure.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function reconfigure {
+    RAW_INVENTORY=/etc/kolla/inventory
+
+    # TODO(jeffrey4l): make some configure file change and
+    # trigger a real reconfigure
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/reconfigure-prechecks
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure >  /tmp/logs/ansible/reconfigure
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-reconfigure
+}
+
+
+reconfigure

tests/run.yml

@@ -145,19 +145,75 @@
       when: scenario == "ceph"
       delegate_to: "primary"
 
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export BASE_DISTRO={{ base_distro }}
-          export INSTALL_TYPE={{ install_type }}
-          export NODEPOOL_TARBALLS_MIRROR=http://{{ zuul_site_mirror_fqdn }}:8080/tarballs
-          export BUILD_IMAGE={{ need_build_image }}
-          export KOLLA_SRC_DIR={{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla
-          export ACTION={{ scenario }}
-
-          tools/setup_gate.sh
+    - name: Run setup_gate.sh script
+      shell:
+        cmd: tools/setup_gate.sh
+        executable: /bin/bash
+        chdir: "{{ kolla_ansible_src_dir }}"
+      environment:
+        BASE_DISTRO: "{{ base_distro }}"
+        INSTALL_TYPE: "{{ install_type }}"
+        NODEPOOL_TARBALLS_MIRROR: "http://{{ zuul_site_mirror_fqdn }}:8080/tarballs"
+        BUILD_IMAGE: "{{ need_build_image }}"
+        KOLLA_SRC_DIR: "{{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla"
+        ACTION: "{{ scenario }}"
+
+    - block:
+        - name: Run deploy.sh script
+          shell:
+            cmd: tests/deploy.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            ACTION: "{{ scenario }}"
+
+        - name: Run test-openstack.sh script
+          shell:
+            cmd: tests/test-openstack.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            ACTION: "{{ scenario }}"
+
+        - name: Run reconfigure.sh script
+          shell:
+            cmd: tests/reconfigure.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            ACTION: "{{ scenario }}"
+
+        - name: Run upgrade.sh script
+          shell:
+            cmd: tests/upgrade.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            ACTION: "{{ scenario }}"
+      when: scenario != "bifrost"
+
+    - block:
+        - name: Run deploy-bifrost.sh script
+          shell:
+            cmd: tests/deploy-bifrost.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+
+        - name: Run test-bifrost.sh script
+          shell:
+            cmd: tests/test-bifrost.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+
+        - name: Run upgrade-bifrost.sh script
+          shell:
+            cmd: tests/upgrade-bifrost.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+      when: scenario == "bifrost"
+
+    - name: Run check-failure.sh script
+      shell:
+        cmd: tests/check-failure.sh
         executable: /bin/bash
         chdir: "{{ kolla_ansible_src_dir }}"
-      delegate_to: "primary"
-      environment: '{{ zuul | zuul_legacy_vars }}'

tests/test-bifrost.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function test_bifrost {
+    # TODO(mgoddard): More testing, deploy bare metal nodes.
+    # TODO(mgoddard): Use openstackclient when clouds.yaml works. See
+    # https://bugs.launchpad.net/bifrost/+bug/1754070.
+    attempts=0
+    while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do
+        attempts=$((attempts + 1))
+        if [[ $attempts -gt 6 ]]; then
+            echo "Timed out waiting for ironic conductor to become active"
+            exit 1
+        fi
+        sleep 10
+    done
+    sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list"
+    sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node"
+    sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node"
+}
+
+
+test_bifrost

tests/test-openstack.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function test_openstack {
+    # Wait for service ready
+    sleep 15
+    . /etc/kolla/admin-openrc.sh
+    # TODO(Jeffrey4l): Restart the memcached container to cleanup all cache.
+    # Remove this after this bug is fixed
+    # https://bugs.launchpad.net/oslo.cache/+bug/1590779
+    sudo docker restart memcached
+    nova --debug service-list
+    openstack --debug network agent list
+    tools/init-runonce
+    nova --debug boot --poll --image $(openstack image list | awk '/cirros/ {print $2}') --nic net-id=$(openstack network list | awk '/demo-net/ {print $2}') --flavor 1 kolla_boot_test
+
+    nova --debug list
+    # If the status is not ACTIVE, print info and exit 1
+    nova --debug show kolla_boot_test | awk '{buf=buf"\n"$0} $2=="status" && $4!="ACTIVE" {failed="yes"}; END {if (failed=="yes") {print buf; exit 1}}'
+    if echo $ACTION | grep -q "ceph"; then
+        openstack volume create --size 2 test_volume
+        openstack server add volume kolla_boot_test test_volume --device /dev/vdb
+    fi
+    if echo $ACTION | grep -q "zun"; then
+        openstack --debug appcontainer service list
+        openstack --debug appcontainer host list
+        # TODO(hongbin): Run a Zun container and assert the container becomes
+        # Running
+    fi
+}
+
+test_openstack

tests/upgrade-bifrost.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function upgrade_bifrost {
+    RAW_INVENTORY=/etc/kolla/inventory
+
+    # TODO(mgoddard): run prechecks.
+    # TODO(mgoddard): add pull action when we have a local registry service in
+    # CI.
+    # TODO(mgoddard): make some configuration file changes and trigger a real
+    # upgrade.
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost >  /tmp/logs/ansible/upgrade-bifrost
+}
+
+
+upgrade_bifrost

tests/upgrade.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+function upgrade {
+    RAW_INVENTORY=/etc/kolla/inventory
+
+    # TODO(jeffrey4l): need run a real upgrade
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/upgrade-prechecks
+    # TODO(mgoddard): add pull action when we have a local registry service in
+    # CI.
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-upgrade
+}
+
+
+upgrade

tools/setup_gate.sh

@@ -122,118 +122,6 @@ function prepare_images {
     popd
 }
 
-function sanity_check {
-    # Wait for service ready
-    sleep 15
-    . /etc/kolla/admin-openrc.sh
-    # TODO(Jeffrey4l): Restart the memcached container to cleanup all cache.
-    # Remove this after this bug is fixed
-    # https://bugs.launchpad.net/oslo.cache/+bug/1590779
-    sudo docker restart memcached
-    nova --debug service-list
-    openstack --debug network agent list
-    tools/init-runonce
-    nova --debug boot --poll --image $(openstack image list | awk '/cirros/ {print $2}') --nic net-id=$(openstack network list | awk '/demo-net/ {print $2}') --flavor 1 kolla_boot_test
-
-    nova --debug list
-    # If the status is not ACTIVE, print info and exit 1
-    nova --debug show kolla_boot_test | awk '{buf=buf"\n"$0} $2=="status" && $4!="ACTIVE" {failed="yes"}; END {if (failed=="yes") {print buf; exit 1}}'
-    if echo $ACTION | grep -q "ceph"; then
-        openstack volume create --size 2 test_volume
-        openstack server add volume kolla_boot_test test_volume --device /dev/vdb
-    fi
-    if echo $ACTION | grep -q "zun"; then
-        openstack --debug appcontainer service list
-        openstack --debug appcontainer host list
-        # TODO(hongbin): Run a Zun container and assert the container becomes
-        # Running
-    fi
-}
-
-function test_openstack {
-    # Create dummy interface for neutron
-    ansible -m shell -i ${RAW_INVENTORY} -b -a "ip l a fake_interface type dummy" all
-
-    #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
-    sudo chmod -R 777 /etc/kolla
-    # Actually do the deployment
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1
-    # TODO(jeffrey4l): add pull action when we have a local registry
-    # service in CI
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy
-
-    # Test OpenStack Environment
-    # TODO: use kolla-ansible check when it's ready
-
-    sanity_check
-
-    # TODO(jeffrey4l): make some configure file change and
-    # trigger a real reconfigure
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure >  /tmp/logs/ansible/reconfigure
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-reconfigure
-    # TODO(jeffrey4l): need run a real upgrade
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-upgrade
-
-    # run prechecks again
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2
-}
-
-function sanity_check_bifrost {
-    # TODO(mgoddard): More testing, deploy bare metal nodes.
-    # TODO(mgoddard): Use openstackclient when clouds.yaml works. See
-    # https://bugs.launchpad.net/bifrost/+bug/1754070.
-    attempts=0
-    while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do
-        attempts=$((attempts + 1))
-        if [[ $attempts -gt 6 ]]; then
-            echo "Timed out waiting for ironic conductor to become active"
-            exit 1
-        fi
-        sleep 10
-    done
-    sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list"
-    sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node"
-    sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node"
-}
-
-function test_bifrost {
-    # TODO(mgoddard): run prechecks.
-
-    # Deploy the bifrost container.
-    # TODO(mgoddard): add pull action when we have a local registry service in
-    # CI.
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost
-
-    # Test Bifrost Environment
-    sanity_check_bifrost
-
-    # TODO(mgoddard): make some configuration file changes and trigger a real
-    # reconfigure.
-    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost >  /tmp/logs/ansible/deploy-bifrost2
-
-    # TODO(mgoddard): perform an upgrade.
-}
-
-check_failure() {
-    # All docker container's status are created, restarting, running, removing,
-    # paused, exited and dead. Containers without running status are treated as
-    # failure. removing is added in docker 1.13, just ignore it now.
-    failed_containers=$(sudo docker ps -a --format "{{.Names}}" \
-        --filter status=created \
-        --filter status=restarting \
-        --filter status=paused \
-        --filter status=exited \
-        --filter status=dead)
-
-    if [[ -n "$failed_containers" ]]; then
-        exit 1;
-    fi
-}
-
-
 
 clone_repos
 setup_ansible
@@ -242,11 +130,3 @@ setup_node
 
 tools/kolla-ansible -i ${RAW_INVENTORY} -e ansible_user=$USER bootstrap-servers > /tmp/logs/ansible/bootstrap-servers
 prepare_images
-
-if [[ $ACTION != bifrost ]]; then
-    test_openstack
-else
-    test_bifrost
-fi
-
-check_failure