Collect description of k8s object to job logs

Change-Id: I15226b9700acf169c100431d5dc7fafe41fbcb87
Signed-off-by: Andrii Ostapenko <andrii.ostapenko@att.com>

playbooks/airship-collect-logs.yaml

@@ -27,6 +27,7 @@
           - airship-gather-libvirt-logs
           - airship-gather-runtime-logs
           - airship-airshipctl-gather-configs
+          - describe-kubernetes-objects
           - airship-gather-pod-logs
     - name: run roles
       include_role:

roles/describe-kubernetes-objects/defaults/main.yaml

@@ -0,0 +1,15 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+kubeconfig: "{{ airshipctl_config_dir_default | default(ansible_env.HOME) }}/.airship/kubeconfig"
+...

roles/describe-kubernetes-objects/tasks/get-contexts.yaml

@@ -0,0 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: "Save kubeconfig contexts"
+  shell: |-
+    kubectl --kubeconfig {{ kubeconfig }} config get-contexts -o name
+  register: kctl_context_command
+
+- name: "Save kubeconfig contexts to list"
+  set_fact:
+    kctl_context_list: "{{ kctl_context_command.stdout.split('\n') }}"
+...

roles/describe-kubernetes-objects/tasks/main.yaml

@@ -0,0 +1,76 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## This task will populate kctl_context_list variable with list of contexts if they exist
+---
+- name: "Set cluster scoped collectibles"
+  set_fact:
+    collect_clustered_objects:
+      - node
+      - clusterrole
+      - clusterrolebinding
+      - storageclass
+      - namespace
+      - persistentvolume
+      - customresourcedefinition
+
+- name: "Set namespace scoped collectibles"
+  set_fact:
+    collect_namespaced_objects:
+      - baremetalhosts
+      - clusters
+      - configmaps
+      - cronjobs
+      - daemonsets
+      - deployments
+      - endpoints
+      - ingresses
+      - jobs
+      - kubeadmconfigs
+      - kubeadmconfigtemplates
+      - kubeadmcontrolplanes
+      - machinedeployments
+      - machines
+      - machinesets
+      - metal3clusters
+      - metal3machines
+      - metal3machinetemplates
+      - networkpolicies
+      - pods
+      - podsecuritypolicies
+      - persistentvolumeclaims
+      - replicasets
+      - rolebindings
+      - roles
+      - secrets
+      - serviceaccounts
+      - services
+      - statefulsets
+
+
+- name: "Get context list"
+  include_tasks: get-contexts.yaml
+
+- name: "Save logs for each context"
+  include_tasks: save-context-objects.yaml
+  when: kctl_context_list | length > 0
+  with_items: "{{ kctl_context_list }}"
+  loop_control:
+    loop_var: kctl_context
+
+- name: "Downloads logs to executor"
+  synchronize:
+    src: "{{ logs_dir }}/objects"
+    dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+    mode: pull
+  ignore_errors: True
+...

roles/describe-kubernetes-objects/tasks/save-context-objects.yaml

@@ -0,0 +1,112 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: "creating directory for cluster scoped objects"
+  file:
+    path: "{{ logs_dir }}/objects/{{ kctl_context }}/cluster"
+    state: directory
+
+- name: "Gathering descriptions for cluster scoped objects"
+  shell: |-
+          set -e
+          export KUBECONFIG="{{ kubeconfig }}"
+          export CONTEXT={{ kctl_context | default("dummy_cluster") }}
+          kubectl config use-context ${CONTEXT}
+          export OBJECT_TYPE="{{ collect_clustered_objects | join(',') }}"
+          export PARALLELISM_FACTOR=2
+
+          function list_objects () {
+            printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "$@"' _ {}
+          }
+          export -f list_objects
+
+          function name_objects () {
+            export OBJECT=$1
+            kubectl get ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${OBJECT} ${1#*/}"' _ {}
+          }
+          export -f name_objects
+
+          function get_objects () {
+            input=($1)
+            export OBJECT=${input[0]}
+            export NAME=${input[1]#*/}
+            echo "${OBJECT}/${NAME}"
+            DIR="{{ logs_dir }}/objects/${CONTEXT}/cluster/${OBJECT}"
+            mkdir -p ${DIR}
+            kubectl get ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml"
+            kubectl describe ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt"
+          }
+          export -f get_objects
+
+          list_objects | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {}
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+
+- name: "creating directory for namespace scoped objects"
+  file:
+    path: "{{ logs_dir }}/objects/{{ kctl_context }}/namespaced"
+    state: directory
+
+- name: "Gathering descriptions for namespace scoped objects"
+  shell: |-
+          set -e
+          export KUBECONFIG="{{ kubeconfig }}"
+          export CONTEXT={{ kctl_context | default("dummy_cluster") }}
+          kubectl config use-context ${CONTEXT}
+          export OBJECT_TYPE="{{ collect_namespaced_objects | join(',') }}"
+          export ALL_OBJECTS_YAML_FILE=$(tempfile)
+          kubectl get all -A -oyaml > ${ALL_OBJECTS_YAML_FILE}
+          export PARALLELISM_FACTOR=2
+
+          function get_namespaces () {
+            kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
+          }
+
+          function list_namespaced_objects () {
+            export NAMESPACE=$1
+            printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} $@"' _ {}
+          }
+          export -f list_namespaced_objects
+
+          function name_objects () {
+            input=($1)
+            export NAMESPACE=${input[0]}
+            export OBJECT=${input[1]}
+            kubectl get -n ${NAMESPACE} ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} ${OBJECT} $@"' _ {}
+          }
+          export -f name_objects
+
+          function get_objects () {
+            input=($1)
+            export NAMESPACE=${input[0]}
+            export OBJECT=${input[1]}
+            export NAME=${input[2]#*/}
+            echo "${NAMESPACE}/${OBJECT}/${NAME}"
+            DIR="{{ logs_dir }}/objects/${CONTEXT}/namespaced/${NAMESPACE}/${OBJECT}"
+            mkdir -p ${DIR}
+            cat "${ALL_OBJECTS_YAML_FILE}" | kustomize cfg grep "metadata.namespace=${NAMESPACE}" | kustomize cfg grep "metadata.name=${OBJECT}" > "${DIR}/${NAME}.yaml"
+            kubectl describe -n ${NAMESPACE} ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt"
+          }
+          export -f get_objects
+
+          get_namespaces | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'list_namespaced_objects "$@"' _ {} | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {}
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+...

zuul.d/jobs.yaml

@@ -134,6 +134,7 @@
         - airship-gather-libvirt-logs
         - airship-gather-runtime-logs
         - airship-airshipctl-gather-configs
+        - describe-kubernetes-objects
         - airship-gather-pod-logs
 
 - job: