diff --git a/scripts/common_functions.sh b/scripts/common_functions.sh index 4e34a3a13..6a6492bfd 100755 --- a/scripts/common_functions.sh +++ b/scripts/common_functions.sh @@ -178,8 +178,14 @@ function canusecache(){ function extract_logs(){ local name=$1 mkdir -p $WORKSPACE/logs/$name - # Exclude journal files because they're large and not useful in a browser - tar -C $WORKSPACE/logs/$name -xf $WORKSPACE/logs/$name.tar.xz var --exclude=journal + local logs_tar="$WORKSPACE/logs/$name.tar.xz" + + if [[ -f $logs_tar ]]; then + # Exclude journal files because they're large and not useful in a browser + tar -C $WORKSPACE/logs/$name -xf $logs_tar var --exclude=journal + else + echo "$logs_tar doesn't exist. Nothing to untar" + fi } function postci(){ @@ -205,6 +211,7 @@ function postci(){ # Generate extra state information from the running undercloud sudo -E $TRIPLEO_ROOT/tripleo-ci/scripts/get_host_info.sh + sudo -E $TRIPLEO_ROOT/tripleo-ci/scripts/get_docker_logs.sh eval $JLOGCMD if [ "$OVB" == "1" ] ; then @@ -224,7 +231,9 @@ function postci(){ NAME=${INSTANCE//=*} ( scp $SSH_OPTIONS $TRIPLEO_ROOT/tripleo-ci/scripts/get_host_info.sh heat-admin@${SANITIZED_ADDRESS}:/tmp + scp $SSH_OPTIONS $TRIPLEO_ROOT/tripleo-ci/scripts/get_docker_logs.sh heat-admin@${SANITIZED_ADDRESS}:/tmp timeout -s 15 -k 600 300 ssh $SSH_OPTIONS heat-admin@$IP sudo /tmp/get_host_info.sh + timeout -s 15 -k 600 300 ssh $SSH_OPTIONS heat-admin@$IP sudo /tmp/get_docker_logs.sh ssh $SSH_OPTIONS heat-admin@$IP $JLOGCMD ssh $SSH_OPTIONS heat-admin@$IP $TARCMD > $WORKSPACE/logs/${NAME}.tar.xz extract_logs $NAME @@ -248,6 +257,8 @@ function postci(){ mkdir $WORKSPACE/logs/subnode-$i/ ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \ sudo $TRIPLEO_ROOT/tripleo-ci/scripts/get_host_info.sh + ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \ + sudo $TRIPLEO_ROOT/tripleo-ci/scripts/get_docker_logs.sh ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip $JLOGCMD ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \ $TARCMD > $WORKSPACE/logs/subnode-$i/subnode-$i.tar.xz @@ -262,6 +273,7 @@ function postci(){ sudo rm -f /etc/sahara/rootwrap.d/sahara.filters ssh $SSH_OPTIONS -i /etc/nodepool/id_rsa $ip \ sudo rm -f /etc/cinder/rootwrap.d/os-brick.filters + let i+=1 done fi diff --git a/scripts/get_docker_logs.sh b/scripts/get_docker_logs.sh new file mode 100755 index 000000000..426d9cdda --- /dev/null +++ b/scripts/get_docker_logs.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +BASE_DOCKER_EXTRA="/var/log/extra/docker" + +if command -v docker && systemctl is-active docker; then + mkdir -p $BASE_DOCKER_EXTRA + ALL_FILE=$BASE_DOCKER_EXTRA/docker_allinfo.log + DOCKER_INFO_CMDS=( + "docker ps --all --size" + "docker images" + "docker volume ls" + "docker stats --all --no-stream" + "docker info" + ) + for cmd in "${DOCKER_INFO_CMDS[@]}"; do + echo "+ $cmd" >> $ALL_FILE + $cmd >> $ALL_FILE + done + + for cont in $(docker ps | awk {'print $NF'} | grep -v NAMES); do + INFO_DIR=$BASE_DOCKER_EXTRA/containers/${cont} + mkdir -p $INFO_DIR + INFO_FILE=$INFO_DIR/docker_info.log + DOCKER_CONTAINER_INFO_CMDS=( + "docker top $cont auxw" + "docker exec $cont top -bwn1" + "docker inspect $cont" + ) + for cmd in "${DOCKER_CONTAINER_INFO_CMDS[@]}"; do + echo "+ $cmd" >> $INFO_FILE + $cmd >> $INFO_FILE + done + docker logs $cont > $INFO_DIR/stdout.log + docker cp $cont:/etc $INFO_DIR/etc + docker cp $cont:/var/lib/kolla/config_files/config.json $INFO_DIR/config.json + # NOTE(flaper87): This should go away. Services should be + # using a `logs` volume + docker cp $cont:/var/log $INFO_DIR/log + + # Delete symlinks because they break log collection and are generally + # not useful + find $INFO_DIR -type l -delete + done + + if [[ -d /var/lib/docker/volumes/logs/_data ]]; then + cp -r /var/lib/docker/volumes/logs/_data $BASE_DOCKER_EXTRA/logs + fi +fi