From 45b2ec99cdbbf19e2463a16500e3423edfe3bc15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Dulko?= <mdulko@redhat.com>
Date: Thu, 22 Mar 2018 11:47:13 +0100
Subject: [PATCH] Add Kuryr pods logs to gate results

When Kuryr is run in pods, logs of its services are not available in the
gate. This commit adds a post-run playbook that will copy information
about Kubernetes resources and logs of all pods into the results.

Change-Id: Idac654854f0b968fe0c3024fd4f0436279a945a5
Partial-Implements: enhance-upstream-gates
---
 .zuul.yaml                   |  3 +++
 playbooks/copy-k8s-logs.yaml | 14 +++++++++++
 tools/gate/copy_k8s_logs.sh  | 45 ++++++++++++++++++++++++++++++++++++
 3 files changed, 62 insertions(+)
 create mode 100644 playbooks/copy-k8s-logs.yaml
 create mode 100755 tools/gate/copy_k8s_logs.sh

diff --git a/.zuul.yaml b/.zuul.yaml
index d0c678a47..baf83fcd1 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -63,6 +63,7 @@
       - openstack/kuryr-tempest-plugin
       - openstack/neutron-lbaas
       - openstack/tempest
+    post-run: playbooks/copy-k8s-logs.yaml
     vars:
       tempest_test_regex: '^(kuryr_tempest_plugin.tests.)'
       tox_envlist: 'all'
@@ -111,6 +112,8 @@
         kuryr-kubernetes: https://git.openstack.org/openstack/kuryr
         devstack-plugin-container: https://git.openstack.org/openstack/devstack-plugin-container
         neutron-lbaas: https://git.openstack.org/openstack/neutron-lbaas
+      zuul_copy_output:
+        '{{ devstack_log_dir }}/kubernetes': 'logs'
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*$
diff --git a/playbooks/copy-k8s-logs.yaml b/playbooks/copy-k8s-logs.yaml
new file mode 100644
index 000000000..866566c34
--- /dev/null
+++ b/playbooks/copy-k8s-logs.yaml
@@ -0,0 +1,14 @@
+- hosts: all
+  tasks:
+    - set_fact:
+        devstack_base_dir: /opt/stack
+      when: devstack_base_dir is not defined
+
+    - name: Copy Kubernetes resources and pods logs
+      shell:
+        cmd: "{{ devstack_base_dir }}/kuryr-kubernetes/tools/gate/copy_k8s_logs.sh"
+        executable: /bin/bash
+        chdir: "{{ zuul.project.src_dir }}"
+      environment:
+        DEVSTACK_BASE_DIR: "{{ devstack_base_dir }}"
+      become: true
diff --git a/tools/gate/copy_k8s_logs.sh b/tools/gate/copy_k8s_logs.sh
new file mode 100755
index 000000000..d4d7fd5b6
--- /dev/null
+++ b/tools/gate/copy_k8s_logs.sh
@@ -0,0 +1,45 @@
+#!/bin/bash -x
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This script takes bits from devstack-gate/functions/cleanup_host in a
+# more generic approach, so we don't need to actually run devstack on the node
+# to cleanup an host.
+
+# Kubernetes resources
+# TODO(dulek): It might be good to split that into Ansible tasks once it's
+#              stable. Until then script is easier to debug and test.
+K8S_LOG_DIR=${DEVSTACK_BASE_DIR}/logs/kubernetes
+mkdir -p ${K8S_LOG_DIR}
+mkdir ${HOME}/.kube
+sudo cp /opt/stack/.kube/config  ${HOME}/.kube/
+sudo chown ${USER}:${USER} ${HOME}/.kube/config
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get pods -o yaml --all-namespaces >> ${K8S_LOG_DIR}/pods.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get svc -o yaml --all-namespaces >> ${K8S_LOG_DIR}/services.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get cm -o yaml --all-namespaces >> ${K8S_LOG_DIR}/configmaps.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get deploy -o yaml --all-namespaces >> ${K8S_LOG_DIR}/deployments.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get ds -o yaml --all-namespaces >> ${K8S_LOG_DIR}/daemonsets.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get nodes -o yaml --all-namespaces >> ${K8S_LOG_DIR}/nodes.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get ingress -o yaml --all-namespaces >> ${K8S_LOG_DIR}/ingress.txt
+
+# Kubernetes pods logs
+mkdir -p ${K8S_LOG_DIR}/pod_logs
+while read -r line
+do
+    name=$(echo ${line} | cut -f1 -d " ")
+    namespace=$(echo ${line} | cut -f2 -d " ")
+    /usr/local/bin/kubectl --kubeconfig=/opt/stack/.kube/config logs -n ${namespace} ${name} >> ${K8S_LOG_DIR}/pod_logs/${namespace}-${name}.txt
+done < <(/usr/local/bin/kubectl get pods -o=custom-columns=NAME:.metadata.name,NAMESPACE:.metadata.namespace --all-namespaces | tail -n +2)
+
+sudo chown -R zuul:zuul ${K8S_LOG_DIR}