From 51a3b05f4ec952d99935fdbbc64347c0404fd29b Mon Sep 17 00:00:00 2001 From: Jiri Stransky <jistr@redhat.com> Date: Wed, 13 Dec 2017 12:33:37 +0100 Subject: [PATCH] Improve debugging possibilities for docker-puppet.py time handling We faced issue where on some environments docker-puppet.py picks up /etc/hosts as modified, even though that shouldn't be the case (LP bug 1709689). On the last occasion we found the cause to be desynced time on the host machine of a virtual setup, and subsequent NTP sync of overcloud nodes (causing a 5 hour skip back in time) racing with docker-puppet.py. Still, more info to debug these kinds of issues would be nice to have. Printing the timestamp of origin_of_time, saving it to a persistent directory and making sure /etc/localtime is mounted into docker-puppet containers should make finding the root cause on such occasions easier. Change-Id: I2ea197673b470379ead295058b6952cce3a69606 Closes-Bug: #1737954 --- docker/docker-puppet.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/docker-puppet.py b/docker/docker-puppet.py index 75a60a422d..4e4a790c58 100755 --- a/docker/docker-puppet.py +++ b/docker/docker-puppet.py @@ -220,7 +220,8 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume # Create a reference timestamp to easily find all files touched by # puppet. The sync ensures we get all the files we want due to # different timestamp. - touch /tmp/the_origin_of_time + origin_of_time=/var/lib/config-data/${NAME}.origin_of_time + touch $origin_of_time sync set +e @@ -246,9 +247,10 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume # Also make a copy of files modified during puppet run # This is useful for debugging + echo "Gathering files modified after $(stat -c '%y' $origin_of_time)" mkdir -p /var/lib/config-data/puppet-generated/${NAME} rsync -a -R -0 --delay-updates --delete-after \ - --files-from=<(find $rsync_srcs -newer /tmp/the_origin_of_time -not -path '/etc/puppet*' -print0) \ + --files-from=<(find $rsync_srcs -newer $origin_of_time -not -path '/etc/puppet*' -print0) \ / /var/lib/config-data/puppet-generated/${NAME} # Write a checksum of the config-data dir, this is used as a @@ -274,6 +276,7 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume '--env', 'HOSTNAME=%s' % short_hostname(), '--env', 'NO_ARCHIVE=%s' % os.environ.get('NO_ARCHIVE', ''), '--env', 'STEP=%s' % os.environ.get('STEP', '6'), + '--volume', '/etc/localtime:/etc/localtime:ro', '--volume', '%s:/etc/config.pp:ro,z' % tmp_man.name, '--volume', '/etc/puppet/:/tmp/puppet-etc/:ro,z', '--volume', '/usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro,z',