Add gate runner script with bash-based execution

Added ability to run deployment scripts directly (not via ansible),
to be able to observe realtime output from commands.

Change-Id: I736b075d6ecf0d7badc1e7703f9238339ab09592
Signed-off-by: Ruslan Aliev <raliev@mirantis.com>

tools/deployment/22_test_configs.sh

@@ -42,7 +42,7 @@ export AIRSHIP_CONFIG_MANIFEST_DIRECTORY=${AIRSHIP_CONFIG_MANIFEST_DIRECTORY:-"/
 export EXTERNAL_KUBECONFIG=${EXTERNAL_KUBECONFIG:-""}
 
 # Remove the contents of the .airship folder, preserving the kustomize plugin directory
-rm -rf $HOME/.airship/config
+rm -rf $HOME/.airship/*config*
 mkdir -p $HOME/.airship
 
 echo "Generate ~/.airship/config and ~/.airship/kubeconfig"

tools/export_sops

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export SOPS_IMPORT_PGP="$(curl -fsSL https://raw.githubusercontent.com/mozilla/sops/master/pgp/sops_functional_tests_key.asc)"
+export SOPS_PGP_FP="FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4"

tools/gate/00_setup.sh

@@ -36,7 +36,7 @@ sudo apt update
 sudo DEBIAN_FRONTEND=noninteractive apt -y install software-properties-common python3-pip curl wget ca-certificates
 sudo DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends install docker.io make
 
-PACKAGES="ansible netaddr"
+PACKAGES="ansible netaddr yq"
 if [[ -z "${http_proxy}" ]]; then
   sudo pip3 install $PACKAGES
 else

tools/gate/20_run_gate_runner.sh

@@ -17,14 +17,98 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -xe
+set -xeo pipefail
+
+source tools/export_sops
 
-TMP_DIR=${TMP_DIR:-"$(dirname $(mktemp -u))"}
-ANSIBLE_HOSTS=${ANSIBLE_HOSTS:-"${TMP_DIR}/ansible_hosts"}
-PLAYBOOK_CONFIG=${PLAYBOOK_CONFIG:-"${TMP_DIR}/config.yaml"}
 export AIRSHIPCTL_WS=${AIRSHIPCTL_WS:-$PWD}
 export AIRSHIP_CONFIG_PHASE_REPO_URL=${AIRSHIP_CONFIG_PHASE_REPO_URL:-$PWD}
 
-sudo -E --preserve-env=AIRSHIPCTL_WS ANSIBLE_STDOUT_CALLBACK=debug ansible-playbook -i "$ANSIBLE_HOSTS" \
+ZUUL_JOBS_PATH=zuul.d/jobs.yaml
-	playbooks/airshipctl-gate-runner.yaml \
+GATE_RUNNER_YAML_PATH=playbooks/airshipctl-gate-runner.yaml
-	-e @"$PLAYBOOK_CONFIG" -v
+OUTPUT_DIR=""
+STOP_SCRIPT=""
+SKIP_LIST=""
+MUTE=0
+
+show_help() {
+cat << EOF
+Usage: $0 [options]
+Run set of deployments scripts for airshipctl
+
+-h,   --help                 Display help
+-s,   --stop-at    NUMBER    Specify script number where to stop execution
+-p    --pass       LIST      Comma separated list of script numbers to skip
+-o,   --output-dir DIRNAME   The output of each script will be saved in the specified directory in a separate file
+-m,   --mute                 Mute the output from scripts
+
+EOF
+}
+
+# read the options
+options=$(getopt -o hmo:p:s: --long help,mute,output-dir:,pass:,stop-at: -- "$@")
+if [ $? != 0 ] ; then echo "Failed to parse options...exiting." >&2 ; exit 1 ; fi
+eval set -- "$options"
+
+while true; do
+  case "$1" in
+  -s | --stop-at)
+      STOP_SCRIPT="$2"
+      shift 2
+      ;;
+  -p | --pass)
+      SKIP_LIST="$2"
+      shift 2
+      ;;
+  -o | --output-dir)
+      OUTPUT_DIR="$2"
+      mkdir -p $OUTPUT_DIR
+      shift 2
+      ;;
+  -m | --mute )
+      MUTE=1
+      shift
+      ;;
+  -h | --help )
+      show_help
+      exit 0
+      ;;
+  -- )
+      shift
+      break
+      ;;
+  esac
+done
+
+SCRIPT_LIST=$(cat $ZUUL_JOBS_PATH | yq '.[9].job.vars.gate_scripts[]' -c -r)
+if [[ ! $SCRIPT_LIST ]]; then
+  SCRIPT_LIST=$(cat $GATE_RUNNER_YAML_PATH | yq '.[0].tasks[0].set_fact.deploy_test_site_scripts_default[]' -c -r)
+fi
+
+SKIP_LIST=$(echo ${SKIP_LIST//,/ })
+
+for script in $SCRIPT_LIST; do
+    SCRIPT_NAME=$(awk -F "/" "{ print \$NF }" <<<$script)
+    if [[ $SCRIPT_NAME =~ ([0-9]+) ]]; then
+      SCRIPT_NUM="${BASH_REMATCH[1]}"
+    fi
+    if [[ " ${SKIP_LIST[@]} " =~ " ${SCRIPT_NUM} " ]]; then
+      if [[ $STOP_SCRIPT ]] && [[ $SCRIPT_NAME =~ "${STOP_SCRIPT}_"* ]]; then
+        break
+      fi
+      continue
+    fi
+
+    echo -e "\033[0;32m[ *** Run script $script *** ] \033[0m "
+    cmd="sudo --preserve-env=AIRSHIPCTL_WS,AIRSHIP_CONFIG_PHASE_REPO_URL,SOPS_IMPORT_PGP,SOPS_PGP_FP $script"
+    if [[ $OUTPUT_DIR ]]; then
+      $cmd > ${OUTPUT_DIR}/${SCRIPT_NAME}.out 2>&1
+    elif [[ "$MUTE" -eq "1" ]]; then
+      $cmd > /dev/null 2>&1
+    else
+      $cmd
+    fi
+    if [[ $STOP_SCRIPT ]] && [[ $SCRIPT_NAME =~ "${STOP_SCRIPT}_"* ]]; then
+      break
+    fi
+done