From 9a9c8c734fb418b32614ea774aad2996ac35d498 Mon Sep 17 00:00:00 2001
From: Jonathan Rosser <jonathan.rosser@rd.bbc.co.uk>
Date: Tue, 14 Mar 2023 14:36:32 +0000
Subject: [PATCH] Refactor log collection from lxc containers

The previous code retrieved logs from the lxc containers by copying
them from the /proc/<pid>/root/... directory which worked except
when the files being collected were symlinks, which would then
potentially point to the wrong, or a missing file on the host itself.

This patch changes the log collection to collect from inside the
running containers using tar, giving a consistent view of the
source files. Any symlinks are replaced with the content of files
they reference in the collected logs.

A side effect of using the more complex lxc-attach/tar approach
was corruption of stdout due to the previous simple method of
parallelising the log collection by running each collection function
in the background. This patch switches to use gnu parallel
as the mechanism to run multiple log collections simultaneously
and keep the console output clean.

Change-Id: Ief03c33b76eac6e256477cfaa16c8a59acd7d702
---
 scripts/log-collect.sh     | 60 +++++++++++++++-----------------------
 scripts/scripts-library.sh |  6 ++--
 2 files changed, 27 insertions(+), 39 deletions(-)

diff --git a/scripts/log-collect.sh b/scripts/log-collect.sh
index e676b21247..5780dce1d9 100755
--- a/scripts/log-collect.sh
+++ b/scripts/log-collect.sh
@@ -94,6 +94,21 @@ function repo_information {
     fi
 }
 
+function store_lxc_artifacts {
+    # Store known artifacts only if they exist. If the target directory does
+    # exist, it will be created.
+    # USAGE: store_lxc_artifacts <container_name> /change/to/dir pattern/to/collect /path/to/store
+    CONTAINER_PID=$(sudo lxc-info -p -n ${1} | awk '{print $2}')
+    CONTAINER_COLLECT="/proc/${CONTAINER_PID}/root/${2}/${3}"
+    if sudo test -e "${CONTAINER_COLLECT}"; then
+      if [[ ! -d "${4}" ]]; then
+          mkdir -vp "${4}"
+      fi
+      echo "Running container artifact sync on ${1} collecting ${3} from dir ${2} to ${4}"
+      sudo lxc-attach -q -n ${1} -- /bin/bash -c "tar --dereference -c -f - -C ${2} ${3} 2>/dev/null | cat" | tar -C ${4} -x 2>/dev/null
+    fi
+}
+
 function store_artifacts {
     # Store known artifacts only if they exist. If the target directory does
     # exist, it will be created.
@@ -153,46 +168,17 @@ store_artifacts /openstack/*repo*/repo/os-releases/*/*/*.txt "${WORKING_DIR}/rep
 # metal path
 store_artifacts /var/www/repo/os-releases/*/*/*.txt "${WORKING_DIR}/repo"
 
-# Gather host etc artifacts
-PIDS=()
-for service in ${COMMON_ETC_LOG_NAMES}; do
-    echo "Running collection for service ${service}"
-    store_artifacts "/etc/${service}" "${WORKING_DIR}/logs/etc/host/" &
-    pid=$!
-    PIDS[${pid}]=${pid}
-    pid=$!
-    PIDS[${pid}]=${pid}
-done
-echo "Waiting for host collection jobs to finish"
-for job_pid in ${!PIDS[@]}; do
-    wait ${PIDS[$job_pid]} || exit 99
-done
-
+# Gather container etc artifacts
+export -f store_artifacts
+IFS=' ' read -a ETC_LOG_NAMES <<< "$COMMON_ETC_LOG_NAMES"
+parallel store_artifacts /etc/{1} ${WORKING_DIR}/logs/etc/host ::: "${ETC_LOG_NAMES[@]}"
 
 # Gather container etc artifacts
+export -f store_lxc_artifacts
 if command -v lxc-ls &> /dev/null; then
-    for CONTAINER_NAME in $(sudo lxc-ls -1); do
-        CONTAINER_PID=$(sudo lxc-info -p -n ${CONTAINER_NAME} | awk '{print $2}')
-        ETC_DIR="/proc/${CONTAINER_PID}/root/etc"
-        LOG_DIR="/proc/${CONTAINER_PID}/root/var/log"
-        repo_information ${CONTAINER_NAME}
-        PIDS=()
-        for service in ${COMMON_ETC_LOG_NAMES}; do
-            echo "Running in container collection for service ${service}"
-            store_artifacts ${ETC_DIR}/${service} "${WORKING_DIR}/logs/etc/openstack/${CONTAINER_NAME}/" &
-            pid=$!
-            PIDS[${pid}]=${pid}
-            store_artifacts ${LOG_DIR}/${service} "${WORKING_DIR}/logs/openstack/${CONTAINER_NAME}/" &
-            pid=$!
-            PIDS[${pid}]=${pid}
-            pid=$!
-            PIDS[${pid}]=${pid}
-        done
-        echo "Waiting for container collection jobs for ${CONTAINER_NAME} to finish"
-        for job_pid in ${!PIDS[@]}; do
-            wait ${PIDS[$job_pid]} || exit 99
-        done
-    done
+  export CONTAINER_NAMES=$(sudo lxc-ls -1)
+  parallel store_lxc_artifacts {1} /etc/ {2} ${WORKING_DIR}/logs/etc/openstack/{1} ::: "${CONTAINER_NAMES[@]}" ::: "${ETC_LOG_NAMES[@]}"
+  parallel store_lxc_artifacts {1} /var/log/ {2} ${WORKING_DIR}/logs/openstack/{1} ::: "${CONTAINER_NAMES[@]}" ::: "${ETC_LOG_NAMES[@]}"
 fi
 
 # gather host and container journals and deprecation warnings
diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh
index 54d4916771..3dbc58be05 100755
--- a/scripts/scripts-library.sh
+++ b/scripts/scripts-library.sh
@@ -206,10 +206,12 @@ function gate_log_requirements {
   case ${DISTRO_ID} in
       ubuntu|debian)
           apt-get update
-          DEBIAN_FRONTEND=noninteractive apt-get -y install iproute2 net-tools
+          DEBIAN_FRONTEND=noninteractive apt-get -y install iproute2 net-tools parallel
           ;;
       rocky|centos|rhel)
-          dnf -y install iproute
+          dnf -y install epel-release
+          sed -i 's/\[epel\]/&\nincludepkgs=parallel/' /etc/yum.repos.d/epel.repo
+          dnf -y --enablerepo=epel install iproute parallel
           ;;
   esac
 }