Prevent corruption of cache by using atomic update

Ensure that cache is written to a temporary file and use an atomic OS
file rename operation to ensure that the cache is either the old
contents or new contents and cannot be corrupted by an exception
occurring during writing or should the process be killed.

Change-Id: I69947cc6d80fdc80ee7addde3a2ff87cd9c3297b

jenkins_jobs/builder.py

@@ -23,6 +23,7 @@ import operator
 import os
 from pprint import pformat
 import re
+import tempfile
 import time
 import xml.etree.ElementTree as XML
 import yaml
@@ -45,8 +46,10 @@ class CacheStorage(object):
     # modules so that they are available to be used when the destructor
     # is being called since python will not guarantee that it won't have
     # removed global module references during teardown.
-    _yaml = yaml
     _logger = logger
+    _os = os
+    _tempfile = tempfile
+    _yaml = yaml
 
     def __init__(self, jenkins_url, flush=False):
         cache_dir = self.get_cache_dir()
@@ -97,23 +100,29 @@ class CacheStorage(object):
         return True
 
     def save(self):
-        # check we initialized sufficiently in case called via __del__
+        # use self references to required modules in case called via __del__
+        # write to tempfile under same directory and then replace to avoid
+        # issues around corruption such the process be killed
+        tfile = self._tempfile.NamedTemporaryFile(dir=self.get_cache_dir(),
+                                                  delete=False)
+        self._yaml.dump(self.data, utils.wrap_stream(tfile))
+        # force contents to be synced on disk before overwriting cachefile
+        tfile.flush()
+        self._os.fsync(tfile.fileno())
+        tfile.close()
+        self._os.rename(tfile.name, self.cachefilename)
+
+        self._logger.debug("Cache written out to '%s'" % self.cachefilename)
+
+    def __del__(self):
+        # check we initialized sufficiently in case called
         # due to an exception occurring in the __init__
         if getattr(self, 'data', None) is not None:
             try:
-                with io.open(self.cachefilename, 'w',
-                             encoding='utf-8') as yfile:
-                    self._yaml.dump(self.data, yfile)
+                self.save()
             except Exception as e:
                 self._logger.error("Failed to write to cache file '%s' on "
                                    "exit: %s" % (self.cachefilename, e))
-            else:
-                self._logger.info("Cache saved")
-                self._logger.debug("Cache written out to '%s'" %
-                                   self.cachefilename)
-
-    def __del__(self):
-        self.save()
 
 
 class Jenkins(object):