From 81efbceea68323579aea1bdea00f4f16d09b4156 Mon Sep 17 00:00:00 2001 From: Andrii Ostapenko Date: Wed, 21 Oct 2020 17:45:52 -0500 Subject: [PATCH] Collect description of k8s object to job logs Change-Id: I15226b9700acf169c100431d5dc7fafe41fbcb87 Signed-off-by: Andrii Ostapenko --- playbooks/airship-collect-logs.yaml | 1 + .../defaults/main.yaml | 15 +++ .../tasks/get-contexts.yaml | 22 ++++ .../tasks/main.yaml | 76 ++++++++++++ .../tasks/save-context-objects.yaml | 112 ++++++++++++++++++ zuul.d/jobs.yaml | 1 + 6 files changed, 227 insertions(+) create mode 100644 roles/describe-kubernetes-objects/defaults/main.yaml create mode 100644 roles/describe-kubernetes-objects/tasks/get-contexts.yaml create mode 100644 roles/describe-kubernetes-objects/tasks/main.yaml create mode 100644 roles/describe-kubernetes-objects/tasks/save-context-objects.yaml diff --git a/playbooks/airship-collect-logs.yaml b/playbooks/airship-collect-logs.yaml index 34cb75aac..3eaaa5823 100644 --- a/playbooks/airship-collect-logs.yaml +++ b/playbooks/airship-collect-logs.yaml @@ -27,6 +27,7 @@ - airship-gather-libvirt-logs - airship-gather-runtime-logs - airship-airshipctl-gather-configs + - describe-kubernetes-objects - airship-gather-pod-logs - name: run roles include_role: diff --git a/roles/describe-kubernetes-objects/defaults/main.yaml b/roles/describe-kubernetes-objects/defaults/main.yaml new file mode 100644 index 000000000..89ae5aeb6 --- /dev/null +++ b/roles/describe-kubernetes-objects/defaults/main.yaml @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +kubeconfig: "{{ airshipctl_config_dir_default | default(ansible_env.HOME) }}/.airship/kubeconfig" +... diff --git a/roles/describe-kubernetes-objects/tasks/get-contexts.yaml b/roles/describe-kubernetes-objects/tasks/get-contexts.yaml new file mode 100644 index 000000000..45321a771 --- /dev/null +++ b/roles/describe-kubernetes-objects/tasks/get-contexts.yaml @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "Save kubeconfig contexts" + shell: |- + kubectl --kubeconfig {{ kubeconfig }} config get-contexts -o name + register: kctl_context_command + +- name: "Save kubeconfig contexts to list" + set_fact: + kctl_context_list: "{{ kctl_context_command.stdout.split('\n') }}" +... diff --git a/roles/describe-kubernetes-objects/tasks/main.yaml b/roles/describe-kubernetes-objects/tasks/main.yaml new file mode 100644 index 000000000..9542d77a4 --- /dev/null +++ b/roles/describe-kubernetes-objects/tasks/main.yaml @@ -0,0 +1,76 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## This task will populate kctl_context_list variable with list of contexts if they exist +--- +- name: "Set cluster scoped collectibles" + set_fact: + collect_clustered_objects: + - node + - clusterrole + - clusterrolebinding + - storageclass + - namespace + - persistentvolume + - customresourcedefinition + +- name: "Set namespace scoped collectibles" + set_fact: + collect_namespaced_objects: + - baremetalhosts + - clusters + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - ingresses + - jobs + - kubeadmconfigs + - kubeadmconfigtemplates + - kubeadmcontrolplanes + - machinedeployments + - machines + - machinesets + - metal3clusters + - metal3machines + - metal3machinetemplates + - networkpolicies + - pods + - podsecuritypolicies + - persistentvolumeclaims + - replicasets + - rolebindings + - roles + - secrets + - serviceaccounts + - services + - statefulsets + + +- name: "Get context list" + include_tasks: get-contexts.yaml + +- name: "Save logs for each context" + include_tasks: save-context-objects.yaml + when: kctl_context_list | length > 0 + with_items: "{{ kctl_context_list }}" + loop_control: + loop_var: kctl_context + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/objects" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True +... diff --git a/roles/describe-kubernetes-objects/tasks/save-context-objects.yaml b/roles/describe-kubernetes-objects/tasks/save-context-objects.yaml new file mode 100644 index 000000000..522b663a9 --- /dev/null +++ b/roles/describe-kubernetes-objects/tasks/save-context-objects.yaml @@ -0,0 +1,112 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "creating directory for cluster scoped objects" + file: + path: "{{ logs_dir }}/objects/{{ kctl_context }}/cluster" + state: directory + +- name: "Gathering descriptions for cluster scoped objects" + shell: |- + set -e + export KUBECONFIG="{{ kubeconfig }}" + export CONTEXT={{ kctl_context | default("dummy_cluster") }} + kubectl config use-context ${CONTEXT} + export OBJECT_TYPE="{{ collect_clustered_objects | join(',') }}" + export PARALLELISM_FACTOR=2 + + function list_objects () { + printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "$@"' _ {} + } + export -f list_objects + + function name_objects () { + export OBJECT=$1 + kubectl get ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${OBJECT} ${1#*/}"' _ {} + } + export -f name_objects + + function get_objects () { + input=($1) + export OBJECT=${input[0]} + export NAME=${input[1]#*/} + echo "${OBJECT}/${NAME}" + DIR="{{ logs_dir }}/objects/${CONTEXT}/cluster/${OBJECT}" + mkdir -p ${DIR} + kubectl get ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml" + kubectl describe ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt" + } + export -f get_objects + + list_objects | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True + +- name: "creating directory for namespace scoped objects" + file: + path: "{{ logs_dir }}/objects/{{ kctl_context }}/namespaced" + state: directory + +- name: "Gathering descriptions for namespace scoped objects" + shell: |- + set -e + export KUBECONFIG="{{ kubeconfig }}" + export CONTEXT={{ kctl_context | default("dummy_cluster") }} + kubectl config use-context ${CONTEXT} + export OBJECT_TYPE="{{ collect_namespaced_objects | join(',') }}" + export ALL_OBJECTS_YAML_FILE=$(tempfile) + kubectl get all -A -oyaml > ${ALL_OBJECTS_YAML_FILE} + export PARALLELISM_FACTOR=2 + + function get_namespaces () { + kubectl get namespaces -o name | awk -F '/' '{ print $NF }' + } + + function list_namespaced_objects () { + export NAMESPACE=$1 + printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} $@"' _ {} + } + export -f list_namespaced_objects + + function name_objects () { + input=($1) + export NAMESPACE=${input[0]} + export OBJECT=${input[1]} + kubectl get -n ${NAMESPACE} ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} ${OBJECT} $@"' _ {} + } + export -f name_objects + + function get_objects () { + input=($1) + export NAMESPACE=${input[0]} + export OBJECT=${input[1]} + export NAME=${input[2]#*/} + echo "${NAMESPACE}/${OBJECT}/${NAME}" + DIR="{{ logs_dir }}/objects/${CONTEXT}/namespaced/${NAMESPACE}/${OBJECT}" + mkdir -p ${DIR} + cat "${ALL_OBJECTS_YAML_FILE}" | kustomize cfg grep "metadata.namespace=${NAMESPACE}" | kustomize cfg grep "metadata.name=${OBJECT}" > "${DIR}/${NAME}.yaml" + kubectl describe -n ${NAMESPACE} ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt" + } + export -f get_objects + + get_namespaces | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'list_namespaced_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True +... diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index e2b7656b1..9f3279aaf 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -134,6 +134,7 @@ - airship-gather-libvirt-logs - airship-gather-runtime-logs - airship-airshipctl-gather-configs + - describe-kubernetes-objects - airship-gather-pod-logs - job: