From 34b5c71e1697a3c8246f2638cc729976388c59ea Mon Sep 17 00:00:00 2001 From: Thirunavukkarasu Palani Date: Mon, 8 Feb 2021 23:46:26 +0000 Subject: [PATCH] Vino collect logs of Kubernetes objects Change-Id: I6224987aacc72f339f631ba9c85d9fc22d4bab4f --- playbooks/vino-collect-logs.yaml | 1 + .../defaults/main.yaml | 15 +++ .../tasks/get-contexts.yaml | 22 ++++ .../tasks/main.yaml | 67 +++++++++++ .../tasks/save-context-objects.yaml | 110 ++++++++++++++++++ .../tasks/save-context-logs.yaml | 1 - 6 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 roles/vino-describe-kubernetes-objects/defaults/main.yaml create mode 100644 roles/vino-describe-kubernetes-objects/tasks/get-contexts.yaml create mode 100644 roles/vino-describe-kubernetes-objects/tasks/main.yaml create mode 100644 roles/vino-describe-kubernetes-objects/tasks/save-context-objects.yaml diff --git a/playbooks/vino-collect-logs.yaml b/playbooks/vino-collect-logs.yaml index 0ff1132..8e8661e 100644 --- a/playbooks/vino-collect-logs.yaml +++ b/playbooks/vino-collect-logs.yaml @@ -19,3 +19,4 @@ LIBVIRT_DEFAULT_URI: qemu:///system roles: - vino-gather-pod-logs + - vino-describe-kubernetes-objects diff --git a/roles/vino-describe-kubernetes-objects/defaults/main.yaml b/roles/vino-describe-kubernetes-objects/defaults/main.yaml new file mode 100644 index 0000000..ca83f9e --- /dev/null +++ b/roles/vino-describe-kubernetes-objects/defaults/main.yaml @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +kubeconfig: "{{ airshipctl_config_dir_default | default(ansible_env.HOME) }}/.kube/config" +... diff --git a/roles/vino-describe-kubernetes-objects/tasks/get-contexts.yaml b/roles/vino-describe-kubernetes-objects/tasks/get-contexts.yaml new file mode 100644 index 0000000..45321a7 --- /dev/null +++ b/roles/vino-describe-kubernetes-objects/tasks/get-contexts.yaml @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "Save kubeconfig contexts" + shell: |- + kubectl --kubeconfig {{ kubeconfig }} config get-contexts -o name + register: kctl_context_command + +- name: "Save kubeconfig contexts to list" + set_fact: + kctl_context_list: "{{ kctl_context_command.stdout.split('\n') }}" +... diff --git a/roles/vino-describe-kubernetes-objects/tasks/main.yaml b/roles/vino-describe-kubernetes-objects/tasks/main.yaml new file mode 100644 index 0000000..e90fc6e --- /dev/null +++ b/roles/vino-describe-kubernetes-objects/tasks/main.yaml @@ -0,0 +1,67 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## This task will populate kctl_context_list variable with list of contexts if they exist +--- +- name: "Set cluster scoped collectibles" + set_fact: + collect_clustered_objects: + - node + - clusterrole + - clusterrolebinding + - storageclass + - namespace + - persistentvolume + - customresourcedefinition + +- name: "Set namespace scoped collectibles" + set_fact: + collect_namespaced_objects: + - baremetalhosts + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - ingresses + - jobs + - networkpolicies + - pods + - podsecuritypolicies + - persistentvolumeclaims + - replicasets + - rolebindings + - roles + - secrets + - serviceaccounts + - services + - statefulsets + - vino + + +- name: "Get context list" + include_tasks: get-contexts.yaml + +- name: "Save logs for each context" + include_tasks: save-context-objects.yaml + when: kctl_context_list | length > 0 + with_items: "{{ kctl_context_list }}" + loop_control: + loop_var: kctl_context + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/objects" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True +... diff --git a/roles/vino-describe-kubernetes-objects/tasks/save-context-objects.yaml b/roles/vino-describe-kubernetes-objects/tasks/save-context-objects.yaml new file mode 100644 index 0000000..a7b1e9e --- /dev/null +++ b/roles/vino-describe-kubernetes-objects/tasks/save-context-objects.yaml @@ -0,0 +1,110 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "creating directory for cluster scoped objects" + file: + path: "{{ logs_dir }}/objects/{{ kctl_context }}/cluster" + state: directory + +- name: "Gathering descriptions for cluster scoped objects" + shell: |- + set -e + export KUBECONFIG="{{ kubeconfig }}" + export CONTEXT={{ kctl_context }} + export OBJECT_TYPE="{{ collect_clustered_objects | join(',') }}" + export PARALLELISM_FACTOR=2 + + function list_objects () { + printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "$@"' _ {} + } + export -f list_objects + + function name_objects () { + export OBJECT=$1 + kubectl get ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${OBJECT} ${1#*/}"' _ {} + } + export -f name_objects + + function get_objects () { + input=($1) + export OBJECT=${input[0]} + export NAME=${input[1]#*/} + echo "${OBJECT}/${NAME}" + DIR="{{ logs_dir }}/objects/${CONTEXT}/cluster/${OBJECT}" + mkdir -p ${DIR} + kubectl get ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml" + kubectl describe ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt" + } + export -f get_objects + + list_objects | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True + +- name: "creating directory for namespace scoped objects" + file: + path: "{{ logs_dir }}/objects/{{ kctl_context }}/namespaced" + state: directory + +- name: "Gathering descriptions for namespace scoped objects" + shell: |- + set -e + export KUBECONFIG="{{ kubeconfig }}" + export CONTEXT={{ kctl_context }} + export OBJECT_TYPE="{{ collect_namespaced_objects | join(',') }}" + export TMP_DIR=$(mktemp -d) + export PARALLELISM_FACTOR=2 + printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P ${PARALLELISM_FACTOR} -n1 bash -c 'kubectl get -A -oyaml "$@" > ${TMP_DIR}/"$@".yaml' _ {} + + function get_namespaces () { + kubectl get namespaces -o name | awk -F '/' '{ print $NF }' + } + + function list_namespaced_objects () { + export NAMESPACE=$1 + printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} $@"' _ {} + } + export -f list_namespaced_objects + + function name_objects () { + input=($1) + export NAMESPACE=${input[0]} + export OBJECT=${input[1]} + kubectl get -n ${NAMESPACE} ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} ${OBJECT} $@"' _ {} + } + export -f name_objects + + function get_objects () { + input=($1) + export NAMESPACE=${input[0]} + export OBJECT=${input[1]} + export NAME=${input[2]#*/} + echo "${NAMESPACE}/${OBJECT}/${NAME}" + DIR="{{ logs_dir }}/objects/${CONTEXT}/namespaced/${NAMESPACE}/${OBJECT}" + mkdir -p ${DIR} + cat "${TMP_DIR}/${OBJECT}.yaml" | kustomize cfg grep "metadata.namespace=${NAMESPACE}" | kustomize cfg grep "metadata.name=${NAME}" > "${DIR}/${NAME}.yaml" + kubectl describe -n ${NAMESPACE} ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt" + } + export -f get_objects + + get_namespaces | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'list_namespaced_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True +... diff --git a/roles/vino-gather-pod-logs/tasks/save-context-logs.yaml b/roles/vino-gather-pod-logs/tasks/save-context-logs.yaml index eee04aa..85d265d 100644 --- a/roles/vino-gather-pod-logs/tasks/save-context-logs.yaml +++ b/roles/vino-gather-pod-logs/tasks/save-context-logs.yaml @@ -49,7 +49,6 @@ done } export -f get_pod_logs - kubectl config use-context {{ kctl_context }} get_namespaces | \ xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pods "$@"' _ {} | \ xargs -r -n 2 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pod_logs "$@"' _ {}