openstack/openstack-ansible
Code Issues Proposed changes

Refactor log collection from lxc containers

Browse Source 
The previous code retrieved logs from the lxc containers by copying
them from the /proc/<pid>/root/... directory which worked except
when the files being collected were symlinks, which would then
potentially point to the wrong, or a missing file on the host itself.

This patch changes the log collection to collect from inside the
running containers using tar, giving a consistent view of the
source files. Any symlinks are replaced with the content of files
they reference in the collected logs.

A side effect of using the more complex lxc-attach/tar approach
was corruption of stdout due to the previous simple method of
parallelising the log collection by running each collection function
in the background. This patch switches to use gnu parallel
as the mechanism to run multiple log collections simultaneously
and keep the console output clean.

Change-Id: Ief03c33b76eac6e256477cfaa16c8a59acd7d702
This commit is contained in:
Jonathan Rosser 2023-03-14 14:36:32 +00:00 committed by Dmitriy Rabotyagov
parent 2e93f05e09
commit 9a9c8c734f
2 changed files with 27 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
scripts/log-collect.sh
View File

@ -94,6 +94,21 @@ function repo_information {
fi
}
function store_lxc_artifacts {
# Store known artifacts only if they exist. If the target directory does
# exist, it will be created.
# USAGE: store_lxc_artifacts <container_name> /change/to/dir pattern/to/collect /path/to/store
CONTAINER_PID=$(sudo lxc-info -p -n ${1} | awk '{print $2}')
CONTAINER_COLLECT="/proc/${CONTAINER_PID}/root/${2}/${3}"
if sudo test -e "${CONTAINER_COLLECT}"; then
if [[ ! -d "${4}" ]]; then
mkdir -vp "${4}"
fi
echo "Running container artifact sync on ${1} collecting ${3} from dir ${2} to ${4}"
sudo lxc-attach -q -n ${1} -- /bin/bash -c "tar --dereference -c -f - -C ${2} ${3} 2>/dev/null | cat" | tar -C ${4} -x 2>/dev/null
fi
}
function store_artifacts {
# Store known artifacts only if they exist. If the target directory does
# exist, it will be created.
@ -153,46 +168,17 @@ store_artifacts /openstack/*repo*/repo/os-releases/*/*/*.txt "${WORKING_DIR}/rep
# metal path
store_artifacts /var/www/repo/os-releases/*/*/*.txt "${WORKING_DIR}/repo"
# Gather host etc artifacts
PIDS=()
for service in ${COMMON_ETC_LOG_NAMES}; do
echo "Running collection for service ${service}"
store_artifacts "/etc/${service}" "${WORKING_DIR}/logs/etc/host/" &
pid=$!
PIDS[${pid}]=${pid}
pid=$!
PIDS[${pid}]=${pid}
done
echo "Waiting for host collection jobs to finish"
for job_pid in ${!PIDS[@]}; do
wait ${PIDS[$job_pid]} || exit 99
done
# Gather container etc artifacts
export -f store_artifacts
IFS=' ' read -a ETC_LOG_NAMES <<< "$COMMON_ETC_LOG_NAMES"
parallel store_artifacts /etc/{1} ${WORKING_DIR}/logs/etc/host ::: "${ETC_LOG_NAMES[@]}"
# Gather container etc artifacts
export -f store_lxc_artifacts
if command -v lxc-ls &> /dev/null; then
for CONTAINER_NAME in $(sudo lxc-ls -1); do
CONTAINER_PID=$(sudo lxc-info -p -n ${CONTAINER_NAME} | awk '{print $2}')
ETC_DIR="/proc/${CONTAINER_PID}/root/etc"
LOG_DIR="/proc/${CONTAINER_PID}/root/var/log"
repo_information ${CONTAINER_NAME}
PIDS=()
for service in ${COMMON_ETC_LOG_NAMES}; do
echo "Running in container collection for service ${service}"
store_artifacts ${ETC_DIR}/${service} "${WORKING_DIR}/logs/etc/openstack/${CONTAINER_NAME}/" &
pid=$!
PIDS[${pid}]=${pid}
store_artifacts ${LOG_DIR}/${service} "${WORKING_DIR}/logs/openstack/${CONTAINER_NAME}/" &
pid=$!
PIDS[${pid}]=${pid}
pid=$!
PIDS[${pid}]=${pid}
done
echo "Waiting for container collection jobs for ${CONTAINER_NAME} to finish"
for job_pid in ${!PIDS[@]}; do
wait ${PIDS[$job_pid]} || exit 99
done
done
export CONTAINER_NAMES=$(sudo lxc-ls -1)
parallel store_lxc_artifacts {1} /etc/ {2} ${WORKING_DIR}/logs/etc/openstack/{1} ::: "${CONTAINER_NAMES[@]}" ::: "${ETC_LOG_NAMES[@]}"
parallel store_lxc_artifacts {1} /var/log/ {2} ${WORKING_DIR}/logs/openstack/{1} ::: "${CONTAINER_NAMES[@]}" ::: "${ETC_LOG_NAMES[@]}"
fi
# gather host and container journals and deprecation warnings

6
scripts/scripts-library.sh
View File

@ -206,10 +206,12 @@ function gate_log_requirements {
case ${DISTRO_ID} in
ubuntu|debian)
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get -y install iproute2 net-tools
DEBIAN_FRONTEND=noninteractive apt-get -y install iproute2 net-tools parallel
;;
rocky|centos|rhel)
dnf -y install iproute
dnf -y install epel-release
sed -i 's/\[epel\]/&\nincludepkgs=parallel/' /etc/yum.repos.d/epel.repo
dnf -y --enablerepo=epel install iproute parallel
;;
esac
}