Add Kuryr pods logs to gate results

When Kuryr is run in pods, logs of its services are not available in the
gate. This commit adds a post-run playbook that will copy information
about Kubernetes resources and logs of all pods into the results.

Change-Id: Idac654854f0b968fe0c3024fd4f0436279a945a5
Partial-Implements: enhance-upstream-gates

.zuul.yaml

@@ -63,6 +63,7 @@
       - openstack/kuryr-tempest-plugin
       - openstack/neutron-lbaas
       - openstack/tempest
+    post-run: playbooks/copy-k8s-logs.yaml
     vars:
       tempest_test_regex: '^(kuryr_tempest_plugin.tests.)'
       tox_envlist: 'all'
@@ -111,6 +112,8 @@
         kuryr-kubernetes: https://git.openstack.org/openstack/kuryr
         devstack-plugin-container: https://git.openstack.org/openstack/devstack-plugin-container
         neutron-lbaas: https://git.openstack.org/openstack/neutron-lbaas
+      zuul_copy_output:
+        '{{ devstack_log_dir }}/kubernetes': 'logs'
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*$

playbooks/copy-k8s-logs.yaml

@@ -0,0 +1,14 @@
+- hosts: all
+  tasks:
+    - set_fact:
+        devstack_base_dir: /opt/stack
+      when: devstack_base_dir is not defined
+
+    - name: Copy Kubernetes resources and pods logs
+      shell:
+        cmd: "{{ devstack_base_dir }}/kuryr-kubernetes/tools/gate/copy_k8s_logs.sh"
+        executable: /bin/bash
+        chdir: "{{ zuul.project.src_dir }}"
+      environment:
+        DEVSTACK_BASE_DIR: "{{ devstack_base_dir }}"
+      become: true

tools/gate/copy_k8s_logs.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -x
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This script takes bits from devstack-gate/functions/cleanup_host in a
+# more generic approach, so we don't need to actually run devstack on the node
+# to cleanup an host.
+
+# Kubernetes resources
+# TODO(dulek): It might be good to split that into Ansible tasks once it's
+#              stable. Until then script is easier to debug and test.
+K8S_LOG_DIR=${DEVSTACK_BASE_DIR}/logs/kubernetes
+mkdir -p ${K8S_LOG_DIR}
+mkdir ${HOME}/.kube
+sudo cp /opt/stack/.kube/config  ${HOME}/.kube/
+sudo chown ${USER}:${USER} ${HOME}/.kube/config
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get pods -o yaml --all-namespaces >> ${K8S_LOG_DIR}/pods.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get svc -o yaml --all-namespaces >> ${K8S_LOG_DIR}/services.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get cm -o yaml --all-namespaces >> ${K8S_LOG_DIR}/configmaps.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get deploy -o yaml --all-namespaces >> ${K8S_LOG_DIR}/deployments.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get ds -o yaml --all-namespaces >> ${K8S_LOG_DIR}/daemonsets.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get nodes -o yaml --all-namespaces >> ${K8S_LOG_DIR}/nodes.txt
+/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get ingress -o yaml --all-namespaces >> ${K8S_LOG_DIR}/ingress.txt
+
+# Kubernetes pods logs
+mkdir -p ${K8S_LOG_DIR}/pod_logs
+while read -r line
+do
+    name=$(echo ${line} | cut -f1 -d " ")
+    namespace=$(echo ${line} | cut -f2 -d " ")
+    /usr/local/bin/kubectl --kubeconfig=/opt/stack/.kube/config logs -n ${namespace} ${name} >> ${K8S_LOG_DIR}/pod_logs/${namespace}-${name}.txt
+done < <(/usr/local/bin/kubectl get pods -o=custom-columns=NAME:.metadata.name,NAMESPACE:.metadata.namespace --all-namespaces | tail -n +2)
+
+sudo chown -R zuul:zuul ${K8S_LOG_DIR}