From a5c6b12e7ed29517963020eea83fcaf7583b2aa8 Mon Sep 17 00:00:00 2001 From: Kostiantyn Kalynovskyi Date: Thu, 10 Sep 2020 14:51:35 -0500 Subject: [PATCH] Allow do gather pod logs for different clusters Relates-To: #346 Closes: #346 Change-Id: I80d16fdafde3d39cdac65942f49e892297cbfeda --- .../defaults/main.yaml | 1 - .../tasks/get-contexts.yaml | 20 +++++++ roles/airship-gather-pod-logs/tasks/main.yaml | 55 +++-------------- .../tasks/save-context-logs.yaml | 59 +++++++++++++++++++ 4 files changed, 88 insertions(+), 47 deletions(-) create mode 100644 roles/airship-gather-pod-logs/tasks/get-contexts.yaml create mode 100644 roles/airship-gather-pod-logs/tasks/save-context-logs.yaml diff --git a/roles/airship-gather-pod-logs/defaults/main.yaml b/roles/airship-gather-pod-logs/defaults/main.yaml index 75185655a..f5f7abb9b 100644 --- a/roles/airship-gather-pod-logs/defaults/main.yaml +++ b/roles/airship-gather-pod-logs/defaults/main.yaml @@ -10,5 +10,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -kctl_context: dummy_cluster kubeconfig: "{{ airshipctl_config_dir_default | default(ansible_env.HOME) }}/.airship/kubeconfig" diff --git a/roles/airship-gather-pod-logs/tasks/get-contexts.yaml b/roles/airship-gather-pod-logs/tasks/get-contexts.yaml new file mode 100644 index 000000000..60886d83f --- /dev/null +++ b/roles/airship-gather-pod-logs/tasks/get-contexts.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + - name: "Save kubeconfig contexts" + shell: |- + kubectl --kubeconfig {{ kubeconfig }} config get-contexts -o name + register: kctl_context_command + + - name: "Save kubeconfig contexts to list" + set_fact: + kctl_context_list: "{{ kctl_context_command.stdout.split('\n') }}" diff --git a/roles/airship-gather-pod-logs/tasks/main.yaml b/roles/airship-gather-pod-logs/tasks/main.yaml index 5f587816b..59a58fa98 100644 --- a/roles/airship-gather-pod-logs/tasks/main.yaml +++ b/roles/airship-gather-pod-logs/tasks/main.yaml @@ -10,53 +10,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +## This task will populate kctl_context_list variable with list of contexts if they exist +- name: "Get context list" + include_tasks: get-contexts.yaml -# NOTE this role has been copied from https://github.com/openstack/openstack-helm-infra/blob/8617c8c1e0ea5fc55d652ccd2a8c2eedf16f69ad/roles/gather-pod-logs/tasks/main.yaml - -- name: "creating directory for pod logs" - file: - path: "{{ logs_dir }}/pod-logs/{{ kctl_context }}" - state: directory - -- name: "creating directory for failed pod logs" - file: - path: "{{ logs_dir }}/pod-logs/{{ kctl_context }}/failed-pods" - state: directory - -- name: "retrieve all container logs, current and previous (if they exist)" - shell: |- - set -e - export KUBECONFIG="{{ kubeconfig }}" - PARALLELISM_FACTOR=2 - function get_namespaces () { - kubectl get namespaces -o name | awk -F '/' '{ print $NF }' - } - function get_pods () { - NAMESPACE=$1 - kubectl get pods -n ${NAMESPACE} -o name | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo ${NAMESPACE} {} - } - export -f get_pods - function get_pod_logs () { - NAMESPACE=${1% *} - POD=${1#* } - INIT_CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o jsonpath='{.spec.initContainers[*].name}') - CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o jsonpath='{.spec.containers[*].name}') - for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do - echo "${NAMESPACE}/${POD}/${CONTAINER}" - mkdir -p "{{ logs_dir }}/pod-logs/{{ kctl_context }}/${NAMESPACE}/${POD}" - mkdir -p "{{ logs_dir }}/pod-logs/{{ kctl_context }}/failed-pods/${NAMESPACE}/${POD}" - kubectl logs ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/{{ kctl_context }}/${NAMESPACE}/${POD}/${CONTAINER}.txt" - kubectl logs --previous ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/{{ kctl_context }}/failed-pods/${NAMESPACE}/${POD}/${CONTAINER}.txt" - done - } - export -f get_pod_logs - kubectl config use-context {{ kctl_context | default("dummy_cluster") }} - get_namespaces | \ - xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pods "$@"' _ {} | \ - xargs -r -n 2 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pod_logs "$@"' _ {} - args: - executable: /bin/bash - ignore_errors: True +- name: "Save logs for each context" + include_tasks: save-context-logs.yaml + when: kctl_context_list | length > 0 + with_items: "{{ kctl_context_list }}" + loop_control: + loop_var: kctl_context - name: "Downloads pod logs to executor" synchronize: diff --git a/roles/airship-gather-pod-logs/tasks/save-context-logs.yaml b/roles/airship-gather-pod-logs/tasks/save-context-logs.yaml new file mode 100644 index 000000000..07f7447c1 --- /dev/null +++ b/roles/airship-gather-pod-logs/tasks/save-context-logs.yaml @@ -0,0 +1,59 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# NOTE this role has been copied from https://github.com/openstack/openstack-helm-infra/blob/8617c8c1e0ea5fc55d652ccd2a8c2eedf16f69ad/roles/gather-pod-logs/tasks/main.yaml + +- name: "creating directory for pod logs" + file: + path: "{{ logs_dir }}/pod-logs/{{ kctl_context }}" + state: directory + +- name: "creating directory for failed pod logs" + file: + path: "{{ logs_dir }}/pod-logs/{{ kctl_context }}/failed-pods" + state: directory + +- name: "retrieve all container logs, current and previous (if they exist)" + shell: |- + set -e + export KUBECONFIG="{{ kubeconfig }}" + PARALLELISM_FACTOR=2 + function get_namespaces () { + kubectl get namespaces -o name | awk -F '/' '{ print $NF }' + } + function get_pods () { + NAMESPACE=$1 + kubectl get pods -n ${NAMESPACE} -o name | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo ${NAMESPACE} {} + } + export -f get_pods + function get_pod_logs () { + NAMESPACE=${1% *} + POD=${1#* } + INIT_CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o jsonpath='{.spec.initContainers[*].name}') + CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o jsonpath='{.spec.containers[*].name}') + for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do + echo "${NAMESPACE}/${POD}/${CONTAINER}" + mkdir -p "{{ logs_dir }}/pod-logs/{{ kctl_context }}/${NAMESPACE}/${POD}" + mkdir -p "{{ logs_dir }}/pod-logs/{{ kctl_context }}/failed-pods/${NAMESPACE}/${POD}" + kubectl logs ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/{{ kctl_context }}/${NAMESPACE}/${POD}/${CONTAINER}.txt" + kubectl logs --previous ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/{{ kctl_context }}/failed-pods/${NAMESPACE}/${POD}/${CONTAINER}.txt" + done + } + export -f get_pod_logs + kubectl config use-context {{ kctl_context | default("dummy_cluster") }} + get_namespaces | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pods "$@"' _ {} | \ + xargs -r -n 2 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pod_logs "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True