From 51a3b05f4ec952d99935fdbbc64347c0404fd29b Mon Sep 17 00:00:00 2001
From: Jiri Stransky <jistr@redhat.com>
Date: Wed, 13 Dec 2017 12:33:37 +0100
Subject: [PATCH] Improve debugging possibilities for docker-puppet.py time
 handling

We faced issue where on some environments docker-puppet.py picks up
/etc/hosts as modified, even though that shouldn't be the case (LP bug
1709689). On the last occasion we found the cause to be desynced time
on the host machine of a virtual setup, and subsequent NTP sync of
overcloud nodes (causing a 5 hour skip back in time) racing with
docker-puppet.py.

Still, more info to debug these kinds of issues would be nice to
have. Printing the timestamp of origin_of_time, saving it to a
persistent directory and making sure /etc/localtime is mounted into
docker-puppet containers should make finding the root cause on such
occasions easier.

Change-Id: I2ea197673b470379ead295058b6952cce3a69606
Closes-Bug: #1737954
---
 docker/docker-puppet.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/docker/docker-puppet.py b/docker/docker-puppet.py
index 75a60a422d..4e4a790c58 100755
--- a/docker/docker-puppet.py
+++ b/docker/docker-puppet.py
@@ -220,7 +220,8 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume
         # Create a reference timestamp to easily find all files touched by
         # puppet. The sync ensures we get all the files we want due to
         # different timestamp.
-        touch /tmp/the_origin_of_time
+        origin_of_time=/var/lib/config-data/${NAME}.origin_of_time
+        touch $origin_of_time
         sync
 
         set +e
@@ -246,9 +247,10 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume
 
             # Also make a copy of files modified during puppet run
             # This is useful for debugging
+            echo "Gathering files modified after $(stat -c '%y' $origin_of_time)"
             mkdir -p /var/lib/config-data/puppet-generated/${NAME}
             rsync -a -R -0 --delay-updates --delete-after \
-                          --files-from=<(find $rsync_srcs -newer /tmp/the_origin_of_time -not -path '/etc/puppet*' -print0) \
+                          --files-from=<(find $rsync_srcs -newer $origin_of_time -not -path '/etc/puppet*' -print0) \
                           / /var/lib/config-data/puppet-generated/${NAME}
 
             # Write a checksum of the config-data dir, this is used as a
@@ -274,6 +276,7 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume
                 '--env', 'HOSTNAME=%s' % short_hostname(),
                 '--env', 'NO_ARCHIVE=%s' % os.environ.get('NO_ARCHIVE', ''),
                 '--env', 'STEP=%s' % os.environ.get('STEP', '6'),
+                '--volume', '/etc/localtime:/etc/localtime:ro',
                 '--volume', '%s:/etc/config.pp:ro,z' % tmp_man.name,
                 '--volume', '/etc/puppet/:/tmp/puppet-etc/:ro,z',
                 '--volume', '/usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro,z',