diff --git a/.zuul.yaml b/.zuul.yaml index baf815551..39fd1fe50 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -19,10 +19,12 @@ jobs: - airship-seaworthy-site-lint-gate - airskiff-site-lint-gate + - airskiff-deploy-gate gate: jobs: - airship-seaworthy-site-lint-gate - airskiff-site-lint-gate + - airskiff-deploy-gate - nodeset: name: airship-treasuremap-single-node @@ -30,6 +32,28 @@ - name: ubuntu-xenial label: ubuntu-xenial +- job: + name: airskiff-deploy-gate + nodeset: airship-treasuremap-single-node + description: | + Deploy Memcached using Airskiff and latest Treasuremap changes. + timeout: 9600 + pre-run: + - tools/gate/playbooks/git-config.yaml + - tools/gate/playbooks/airskiff-reduce-site.yaml + run: tools/gate/playbooks/airskiff-deploy-gate.yaml + post-run: tools/gate/playbooks/debug-report.yaml + required-projects: + - openstack/airship-armada + - openstack/airship-deckhand + - openstack/airship-pegleg + - openstack/airship-shipyard + - openstack/openstack-helm + - openstack/openstack-helm-infra + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ + - job: name: airship-seaworthy-site-lint-gate nodeset: airship-treasuremap-single-node diff --git a/tools/gate/debug-report.sh b/tools/gate/debug-report.sh new file mode 100755 index 000000000..d277ec0f2 --- /dev/null +++ b/tools/gate/debug-report.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +set -ux + +SUBDIR_NAME=debug-$(hostname) + +# NOTE(mark-burnett): This should add calicoctl to the path. +export PATH=${PATH}:/opt/cni/bin + +TEMP_DIR=$(mktemp -d) +export TEMP_DIR +export BASE_DIR="${TEMP_DIR}/${SUBDIR_NAME}" +export HELM_DIR="${BASE_DIR}/helm" +export CALICO_DIR="${BASE_DIR}/calico" + +mkdir -p "${BASE_DIR}" + +PARALLELISM_FACTOR=2 + +function get_namespaces () { + kubectl get namespaces -o name | awk -F '/' '{ print $NF }' +} + +function get_pods () { + NAMESPACE=$1 + kubectl get pods -n "${NAMESPACE}" -o name --show-all | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo "${NAMESPACE}" {} +} +export -f get_pods + +function get_pod_logs () { + NAMESPACE=${1% *} + POD=${1#* } + INIT_CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.initContainers[]?.name') + CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.containers[].name') + POD_DIR="${BASE_DIR}/pod-logs/${NAMESPACE}/${POD}" + mkdir -p "${POD_DIR}" + for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do + kubectl logs "${POD}" -n "${NAMESPACE}" -c "${CONTAINER}" > "${POD_DIR}/${CONTAINER}.txt" + done +} +export -f get_pod_logs + +function list_namespaced_objects () { + export NAMESPACE=$1 + cat | awk "{ print \"${NAMESPACE} \" \$1 }" < "${DIR}/${NAME}.yaml" +} +export -f get_objects + +function get_releases () { + helm list --all --short +} + +function get_release () { + input=($1) + RELEASE=${input[0]} + helm status "${RELEASE}" > "${HELM_DIR}/${RELEASE}.txt" + +} +export -f get_release + +if which helm; then + mkdir -p "${HELM_DIR}" + helm list --all > "${HELM_DIR}/list" & + get_releases | \ + xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_release "$@"' _ {} +fi + +kubectl get --all-namespaces -o wide pods > "${BASE_DIR}/pods.txt" & + +get_namespaces | \ + xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pods "$@"' _ {} | \ + xargs -r -n 2 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pod_logs "$@"' _ {} & + +get_namespaces | \ + xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'list_namespaced_objects "$@"' _ {} | \ + xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'name_objects "$@"' _ {} | \ + xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_objects "$@"' _ {} & + +iptables-save > "${BASE_DIR}/iptables" & + +if which calicoctl; then + mkdir -p "${CALICO_DIR}" + for kind in bgpPeer hostEndpoint ipPool nodes policy profile workloadEndpoint; do + calicoctl get "${kind}" -o yaml > "${CALICO_DIR}/${kind}.yaml" & + done +fi + +wait + +tar zcf "${SUBDIR_NAME}.tgz" -C "${TEMP_DIR}" "${SUBDIR_NAME}" diff --git a/tools/gate/manifests/full-site.yaml b/tools/gate/manifests/full-site.yaml new file mode 100644 index 000000000..52b3b4c2f --- /dev/null +++ b/tools/gate/manifests/full-site.yaml @@ -0,0 +1,21 @@ +--- +schema: armada/Manifest/v1 +metadata: + schema: metadata/Document/v1 + name: full-site + replacement: true + layeringDefinition: + abstract: false + layer: site + parentSelector: + name: full-site-global + actions: + - method: replace + path: .chart_groups + storagePolicy: cleartext +data: + release_prefix: airship + chart_groups: + - openstack-ingress-controller-airskiff + - openstack-memcached +... diff --git a/tools/gate/playbooks/airskiff-deploy-gate.yaml b/tools/gate/playbooks/airskiff-deploy-gate.yaml new file mode 100644 index 000000000..d796413d5 --- /dev/null +++ b/tools/gate/playbooks/airskiff-deploy-gate.yaml @@ -0,0 +1,79 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: ubuntu-xenial + tasks: + - name: Install required packages + shell: | + ./tools/deployment/airskiff/developer/000-install-packages.sh + args: + chdir: "{{ zuul.project.src_dir }}" + become: yes + + - name: Build Armada + shell: | + make images + args: + chdir: "{{ zuul.projects['git.openstack.org/openstack/airship-armada'].src_dir }}" + become: yes + + - name: Build Deckhand + shell: | + make + args: + chdir: "{{ zuul.projects['git.openstack.org/openstack/airship-deckhand'].src_dir }}" + become: yes + + - name: Build Shipyard + shell: | + make + args: + chdir: "{{ zuul.projects['git.openstack.org/openstack/airship-shipyard'].src_dir }}" + become: yes + + - name: Deploy Kubernetes with KubeADM + shell: | + ./tools/deployment/airskiff/developer/010-deploy-k8s.sh + args: + chdir: "{{ zuul.project.src_dir }}" + + - name: Setup OpenStack Client + shell: | + ./tools/deployment/airskiff/developer/020-setup-client.sh + args: + chdir: "{{ zuul.project.src_dir }}" + become: yes + + - name: Deploy Airship components using Armada + shell: | + mkdir ~/.kube + cp -rp /home/zuul/.kube/config ~/.kube/config + ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh + args: + chdir: "{{ zuul.project.src_dir }}" + become: yes + + - name: Deploy Software using Airship + shell: | + ./tools/deployment/airskiff/developer/100-deploy-osh.sh + args: + chdir: "{{ zuul.project.src_dir }}" + become: yes + + - name: Wait for deployment completion + shell: | + ./tools/gate/wait-for-shipyard.sh + args: + chdir: "{{ zuul.project.src_dir }}" + become: yes diff --git a/tools/gate/playbooks/airskiff-reduce-site.yaml b/tools/gate/playbooks/airskiff-reduce-site.yaml new file mode 100644 index 000000000..ba8181272 --- /dev/null +++ b/tools/gate/playbooks/airskiff-reduce-site.yaml @@ -0,0 +1,22 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: ubuntu-xenial + tasks: + - name: Replace Armada manifest + shell: | + mv tools/gate/manifests/full-site.yaml \ + site/airskiff/software/manifests/full-site.yaml + args: + chdir: "{{ zuul.project.src_dir }}" diff --git a/tools/gate/playbooks/debug-report.yaml b/tools/gate/playbooks/debug-report.yaml new file mode 100644 index 000000000..45714035c --- /dev/null +++ b/tools/gate/playbooks/debug-report.yaml @@ -0,0 +1,34 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: ubuntu-xenial + tasks: + - name: Build debug report + shell: | + ./tools/gate/debug-report.sh + args: + chdir: "{{ zuul.project.src_dir }}" + + - name: Extract debug report + shell: | + mkdir -p /tmp/debug + tar -xf debug-* -C /tmp/debug + args: + chdir: "{{ zuul.project.src_dir }}" + + - name: Pull logs to executor + synchronize: + src: /tmp/debug + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull diff --git a/tools/gate/wait-for-shipyard.sh b/tools/gate/wait-for-shipyard.sh new file mode 100755 index 000000000..f949f3bd4 --- /dev/null +++ b/tools/gate/wait-for-shipyard.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License + +set -e + +: "${SHIPYARD:=../airship-shipyard/tools/shipyard.sh}" + +ACTION=$(${SHIPYARD} get actions | grep -i "Processing" | awk '{ print $2 }') + +echo -e "\nWaiting for $ACTION..." + +while true; do + status=$(${SHIPYARD} describe "$ACTION" | grep -i "Lifecycle" | \ + awk '{print $2}') + + if [ "${status}" == "Failed" ]; then + echo -e "\n$ACTION FAILED\n" + ${SHIPYARD} describe "${ACTION}" + exit 1 + fi + + if [ "${status}" == "Complete" ]; then + echo -e "\n$ACTION completed SUCCESSFULLY\n" + ${SHIPYARD} describe "${ACTION}" + exit 0 + fi + + sleep 10 +done