Browse Source

Add lightweight debug report for gate-multinode

Change-Id: Ida4b7917772b5c46d898c308328765ad79dda9f4
Kaspars Skels 2 months ago
parent
commit
9cd5654b48

+ 105
- 0
tools/multi_nodes_gate/airship_gate/bin/debug-report-lite.sh View File

@@ -0,0 +1,105 @@
1
+#!/usr/bin/env bash
2
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+#     http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+
16
+
17
+set -ux
18
+
19
+SUBDIR_NAME=debug-$(hostname)
20
+
21
+# NOTE(mark-burnett): This should add calicoctl to the path.
22
+export PATH=${PATH}:/opt/cni/bin
23
+
24
+TEMP_DIR=$(mktemp -d)
25
+export TEMP_DIR
26
+export BASE_DIR="${TEMP_DIR}/${SUBDIR_NAME}"
27
+export HELM_DIR="${BASE_DIR}/helm"
28
+export CALICO_DIR="${BASE_DIR}/calico"
29
+
30
+mkdir -p "${BASE_DIR}"
31
+
32
+PARALLELISM_FACTOR=2
33
+
34
+function get_namespaces () {
35
+    kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
36
+}
37
+
38
+function get_pods () {
39
+    NAMESPACE=$1
40
+    kubectl get pods -n "${NAMESPACE}" -o name --show-all | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo "${NAMESPACE}" {}
41
+}
42
+export -f get_pods
43
+
44
+function get_pod_logs () {
45
+    NAMESPACE=${1% *}
46
+    POD=${1#* }
47
+    INIT_CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.initContainers[]?.name')
48
+    CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.containers[].name')
49
+    POD_DIR="${BASE_DIR}/pod-logs/${NAMESPACE}/${POD}"
50
+    mkdir -p "${POD_DIR}"
51
+    for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do
52
+        echo "get_pod_logs() [${NAMESPACE}] ${POD} ${CONTAINER}"
53
+        kubectl logs "${POD}" -n "${NAMESPACE}" -c "${CONTAINER}" > "${POD_DIR}/${CONTAINER}.txt"
54
+    done
55
+}
56
+
57
+
58
+export -f get_pod_logs
59
+
60
+function get_releases () {
61
+    helm list --all --short
62
+}
63
+
64
+function get_release () {
65
+    input=($1)
66
+    RELEASE=${input[0]}
67
+    helm status "${RELEASE}" > "${HELM_DIR}/${RELEASE}.txt"
68
+
69
+}
70
+export -f get_release
71
+
72
+if which helm; then
73
+    mkdir -p "${HELM_DIR}"
74
+    helm list --all > "${HELM_DIR}/list"
75
+    get_releases | \
76
+        xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_release "$@"' _ {}
77
+fi
78
+
79
+kubectl get --all-namespaces -o wide pods > "${BASE_DIR}/pods.txt"
80
+
81
+get_namespaces | \
82
+    xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pods "$@"' _ {} | \
83
+    xargs -r -n 2 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pod_logs "$@"' _ {}
84
+
85
+iptables-save > "${BASE_DIR}/iptables"
86
+
87
+cat /var/log/syslog > "${BASE_DIR}/syslog"
88
+cat /var/log/armada/bootstrap-armada.log > "${BASE_DIR}/bootstrap-armada.log"
89
+
90
+ip addr show > "${BASE_DIR}/ifconfig"
91
+ip route show > "${BASE_DIR}/ip-route"
92
+cp -p /etc/resolv.conf "${BASE_DIR}/"
93
+
94
+env | sort --ignore-case > "${BASE_DIR}/environment"
95
+docker images > "${BASE_DIR}/docker-images"
96
+
97
+if which calicoctl; then
98
+    mkdir -p "${CALICO_DIR}"
99
+    for kind in bgpPeer hostEndpoint ipPool nodes policy profile workloadEndpoint; do
100
+        calicoctl get "${kind}" -o yaml > "${CALICO_DIR}/${kind}.yaml"
101
+    done
102
+fi
103
+
104
+tar zcf "${SUBDIR_NAME}.tgz" -C "${TEMP_DIR}" "${SUBDIR_NAME}"
105
+

+ 31
- 0
tools/multi_nodes_gate/airship_gate/stages/debug-report-lite.sh View File

@@ -0,0 +1,31 @@
1
+#!/usr/bin/env bash
2
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+#     http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+
16
+set -ex
17
+
18
+source "${GATE_UTILS}"
19
+
20
+ssh_cmd "${GENESIS_NAME}" apt-get install -y jq
21
+
22
+# Copies script and virtmgr private key to genesis VM
23
+ssh_cmd "${GENESIS_NAME}" mkdir -p /root/airship
24
+rsync_cmd "${REPO_ROOT}/tools/multi_nodes_gate/airship_gate/bin/debug-report-lite.sh" "${GENESIS_NAME}:/root/airship/"
25
+
26
+set -o pipefail
27
+ssh_cmd "${GENESIS_NAME}" /root/airship/debug-report-lite.sh 2>&1 | tee -a "${LOG_FILE}"
28
+set +o pipefail
29
+
30
+rsync_cmd "${GENESIS_NAME}:/root/debug-${GENESIS_NAME}.tgz" .
31
+

Loading…
Cancel
Save