openstack/tripleo-ci
Code Issues Proposed changes

Collect more info about docker

Browse Source 
Include volumes list and the content of some of the internal dirs in the
containers. This will provide more info about the configs mounted in the
container and their logs.

Co-Authored-By: Martin André <m.andre@redhat.com>
Change-Id: Ie5872d31dc6caf566dda7aa78b14502708babe4c
This commit is contained in:
Flavio Percoco 2017-03-03 11:46:17 +01:00 committed by Martin André
parent a952523319
commit 94cd0f876b
2 changed files with 62 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scripts/common_functions.sh
View File

@ -178,8 +178,14 @@ function canusecache(){
function extract_logs(){
local name=$1
mkdir -p $WORKSPACE/logs/$name
local logs_tar="$WORKSPACE/logs/$name.tar.xz"
if [[ -f $logs_tar ]]; then
# Exclude journal files because they're large and not useful in a browser
tar -C $WORKSPACE/logs/$name -xf $WORKSPACE/logs/$name.tar.xz var --exclude=journal
tar -C $WORKSPACE/logs/$name -xf $logs_tar var --exclude=journal
else
echo "$logs_tar doesn't exist. Nothing to untar"
fi
}
function postci(){
@ -205,6 +211,7 @@ function postci(){
# Generate extra state information from the running undercloud
sudo -E $TRIPLEO_ROOT/tripleo-ci/scripts/get_host_info.sh
sudo -E $TRIPLEO_ROOT/tripleo-ci/scripts/get_docker_logs.sh
eval $JLOGCMD
if [ "$OVB" == "1" ] ; then
@ -224,7 +231,9 @@ function postci(){
NAME=${INSTANCE//=*}
(
scp $SSH_OPTIONS $TRIPLEO_ROOT/tripleo-ci/scripts/get_host_info.sh heat-admin@${SANITIZED_ADDRESS}:/tmp
scp $SSH_OPTIONS $TRIPLEO_ROOT/tripleo-ci/scripts/get_docker_logs.sh heat-admin@${SANITIZED_ADDRESS}:/tmp
timeout -s 15 -k 600 300 ssh $SSH_OPTIONS heat-admin@$IP sudo /tmp/get_host_info.sh
timeout -s 15 -k 600 300 ssh $SSH_OPTIONS heat-admin@$IP sudo /tmp/get_docker_logs.sh
ssh $SSH_OPTIONS heat-admin@$IP $JLOGCMD
ssh $SSH_OPTIONS heat-admin@$IP $TARCMD > $WORKSPACE/logs/${NAME}.tar.xz
extract_logs $NAME
@ -248,6 +257,8 @@ function postci(){
mkdir $WORKSPACE/logs/subnode-$i/
ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
sudo $TRIPLEO_ROOT/tripleo-ci/scripts/get_host_info.sh
ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
sudo $TRIPLEO_ROOT/tripleo-ci/scripts/get_docker_logs.sh
ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip $JLOGCMD
ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
$TARCMD > $WORKSPACE/logs/subnode-$i/subnode-$i.tar.xz
@ -262,6 +273,7 @@ function postci(){
sudo rm -f /etc/sahara/rootwrap.d/sahara.filters
ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \
sudo rm -f /etc/cinder/rootwrap.d/os-brick.filters
let i+=1
done
fi

48
scripts/get_docker_logs.sh Executable file
View File

@ -0,0 +1,48 @@
#!/bin/bash
BASE_DOCKER_EXTRA="/var/log/extra/docker"
if command -v docker && systemctl is-active docker; then
mkdir -p $BASE_DOCKER_EXTRA
ALL_FILE=$BASE_DOCKER_EXTRA/docker_allinfo.log
DOCKER_INFO_CMDS=(
"docker ps --all --size"
"docker images"
"docker volume ls"
"docker stats --all --no-stream"
"docker info"
)
for cmd in "${DOCKER_INFO_CMDS[@]}"; do
echo "+ $cmd" >> $ALL_FILE
$cmd >> $ALL_FILE
done
for cont in $(docker ps | awk {'print $NF'} | grep -v NAMES); do
INFO_DIR=$BASE_DOCKER_EXTRA/containers/${cont}
mkdir -p $INFO_DIR
INFO_FILE=$INFO_DIR/docker_info.log
DOCKER_CONTAINER_INFO_CMDS=(
"docker top $cont auxw"
"docker exec $cont top -bwn1"
"docker inspect $cont"
)
for cmd in "${DOCKER_CONTAINER_INFO_CMDS[@]}"; do
echo "+ $cmd" >> $INFO_FILE
$cmd >> $INFO_FILE
done
docker logs $cont > $INFO_DIR/stdout.log
docker cp $cont:/etc $INFO_DIR/etc
docker cp $cont:/var/lib/kolla/config_files/config.json $INFO_DIR/config.json
# NOTE(flaper87): This should go away. Services should be
# using a `logs` volume
docker cp $cont:/var/log $INFO_DIR/log
# Delete symlinks because they break log collection and are generally
# not useful
find $INFO_DIR -type l -delete
done
if [[ -d /var/lib/docker/volumes/logs/_data ]]; then
cp -r /var/lib/docker/volumes/logs/_data $BASE_DOCKER_EXTRA/logs
fi
fi