Move default cache dir to persistent location

While this is called a "cache", it is important for it to survive. On
reboot, servers may need what was in the cfn config to restore complex
network configurations.

We introduce a new command line option, --backup-cachedir, that will
default to the old path, /var/run/os-collect-config. This will keep
things working for any tools that have been hard coded to use the old
path.

Change-Id: I78b3851b35addfc16913e3cd53c9d0e7eb3d191a

README.rst

@@ -39,7 +39,7 @@ set as a colon separated list in the environment variable
 above, *os-refresh-config* would be executed with something like this
 in *OS_CONFIG_FILES*::
 
-  /var/run/os-collect-config/ec2.json:/var/run/os-collect-config/cfn.json
+  /var/lib/os-collect-config/ec2.json:/var/lib/os-collect-config/cfn.json
 
 The previous version of the metadata from a source (if available) is present at $FILENAME.last.
 

os_collect_config/collect.py

@@ -16,6 +16,7 @@
 import hashlib
 import json
 import os
+import shutil
 import signal
 import subprocess
 import sys
@@ -39,8 +40,11 @@ opts = [
                     ' not specified, os-collect-config will print the'
                     ' collected data as a json map and exit.'),
     cfg.StrOpt('cachedir',
-               default='/var/run/os-collect-config',
+               default='/var/lib/os-collect-config',
                help='Directory in which to store local cache of metadata'),
+    cfg.StrOpt('backup-cachedir',
+               default='/var/run/os-collect-config',
+               help='Copy cache contents to this directory as well.'),
     cfg.MultiStrOpt(
         'collectors',
         positional=True,
@@ -128,6 +132,10 @@ def collect_all(collectors, store=False, requests_impl_map=None):
 
     if any_changed:
         cache.store_meta_list('os_config_files', collectors)
+        if os.path.exists(CONF.backup_cachedir):
+            shutil.rmtree(CONF.backup_cachedir)
+        if os.path.exists(CONF.cachedir):
+            shutil.copytree(CONF.cachedir, CONF.backup_cachedir)
     return (any_changed, paths_or_content)
 
 

os_collect_config/tests/test_collect.py

@@ -69,6 +69,7 @@ class TestCollect(testtools.TestCase):
     def test_main(self):
         expected_cmd = self.getUniqueString()
         cache_dir = self.useFixture(fixtures.TempDir())
+        backup_cache_dir = self.useFixture(fixtures.TempDir())
         fake_metadata = _setup_local_metadata(self)
         occ_args = [
             'os-collect-config',
@@ -76,6 +77,8 @@ class TestCollect(testtools.TestCase):
             expected_cmd,
             '--cachedir',
             cache_dir.path,
+            '--backup-cachedir',
+            backup_cache_dir.path,
             '--config-file',
             '/dev/null',
             '--cfn-metadata-url',
@@ -94,32 +97,35 @@ class TestCollect(testtools.TestCase):
         calls = self._fake_popen_call_main(occ_args)
         proc_args = calls[0]
         self.assertEqual(expected_cmd, proc_args['args'])
-        list_path = os.path.join(cache_dir.path, 'os_config_files.json')
-        with open(list_path) as list_file:
-            config_list = json.loads(list_file.read())
-        self.assertThat(config_list, matchers.IsInstance(list))
-        env_config_list = proc_args['env']['OS_CONFIG_FILES'].split(':')
-        self.assertEqual(env_config_list, config_list)
-        keys_found = set()
-        for path in env_config_list:
-            self.assertTrue(os.path.exists(path))
-            with open(path) as cfg_file:
-                contents = json.loads(cfg_file.read())
-                keys_found.update(set(contents.keys()))
-        # From test_ec2.FakeRequests
-        self.assertIn("local-ipv4", keys_found)
-        self.assertIn("reservation-id", keys_found)
-        # From test_cfn.FakeRequests
-        self.assertIn("int1", keys_found)
-        self.assertIn("map_ab", keys_found)
+        for test_dir in (cache_dir, backup_cache_dir):
+            list_path = os.path.join(test_dir.path, 'os_config_files.json')
+            with open(list_path) as list_file:
+                config_list = json.loads(list_file.read())
+            self.assertThat(config_list, matchers.IsInstance(list))
+            env_config_list = proc_args['env']['OS_CONFIG_FILES'].split(':')
+            self.assertEqual(env_config_list, config_list)
+            keys_found = set()
+            for path in env_config_list:
+                self.assertTrue(os.path.exists(path))
+                with open(path) as cfg_file:
+                    contents = json.loads(cfg_file.read())
+                    keys_found.update(set(contents.keys()))
+            # From test_ec2.FakeRequests
+            self.assertIn("local-ipv4", keys_found)
+            self.assertIn("reservation-id", keys_found)
+            # From test_cfn.FakeRequests
+            self.assertIn("int1", keys_found)
+            self.assertIn("map_ab", keys_found)
 
     def test_main_force_command(self):
         cache_dir = self.useFixture(fixtures.TempDir())
+        backup_cache_dir = self.useFixture(fixtures.TempDir())
         fake_metadata = _setup_local_metadata(self)
         occ_args = [
             'os-collect-config',
             '--command', 'foo',
             '--cachedir', cache_dir.path,
+            '--backup-cachedir', backup_cache_dir.path,
             '--config-file', '/dev/null',
             '--heat_local-path', fake_metadata,
             '--force',
@@ -133,6 +139,7 @@ class TestCollect(testtools.TestCase):
 
     def test_main_command_failed_no_caching(self):
         cache_dir = self.useFixture(fixtures.TempDir())
+        backup_cache_dir = self.useFixture(fixtures.TempDir())
         fake_metadata = _setup_local_metadata(self)
         occ_args = [
             'os-collect-config',
@@ -140,6 +147,8 @@ class TestCollect(testtools.TestCase):
             'foo',
             '--cachedir',
             cache_dir.path,
+            '--backup-cachedir',
+            backup_cache_dir.path,
             '--config-file',
             '/dev/null',
             '--heat_local-path',
@@ -152,9 +161,10 @@ class TestCollect(testtools.TestCase):
             return dict(returncode=1)
         self.useFixture(fixtures.FakePopen(capture_popen))
         self._call_main(occ_args)
-        cache_contents = os.listdir(cache_dir.path)
-        last_files = [name for name in cache_contents if name.endswith('last')]
-        self.assertEqual([], last_files)
+        for test_dir in (cache_dir, backup_cache_dir):
+            cache_contents = os.listdir(test_dir.path)
+            last_files = [n for n in cache_contents if n.endswith('last')]
+            self.assertEqual([], last_files)
 
     def test_main_no_command(self):
         fake_args = [
@@ -202,11 +212,13 @@ class TestCollect(testtools.TestCase):
 
     def test_main_print_only(self):
         cache_dir = self.useFixture(fixtures.TempDir())
+        backup_cache_dir = self.useFixture(fixtures.TempDir())
         fake_metadata = _setup_local_metadata(self)
         args = [
             'os-collect-config',
             '--command', 'bar',
             '--cachedir', cache_dir.path,
+            '--backup-cachedir', backup_cache_dir.path,
             '--config-file', '/dev/null',
             '--print',
             '--cfn-metadata-url',
@@ -269,6 +281,7 @@ class TestCollectAll(testtools.TestCase):
         self.useFixture(fixtures.FakeLogger())
         collect.setup_conf()
         self.cache_dir = self.useFixture(fixtures.TempDir())
+        self.backup_cache_dir = self.useFixture(fixtures.TempDir())
         self.clean_conf = copy.copy(cfg.CONF)
 
         def restore_copy():
@@ -276,6 +289,7 @@ class TestCollectAll(testtools.TestCase):
         self.addCleanup(restore_copy)
 
         cfg.CONF.cachedir = self.cache_dir.path
+        cfg.CONF.backup_cachedir = self.backup_cache_dir.path
         cfg.CONF.cfn.metadata_url = 'http://127.0.0.1:8000/v1/'
         cfg.CONF.cfn.stack_name = 'foo'
         cfg.CONF.cfn.path = ['foo.Metadata']
@@ -339,7 +353,7 @@ class TestConf(testtools.TestCase):
 
     def test_setup_conf(self):
         collect.setup_conf()
-        self.assertEqual('/var/run/os-collect-config', cfg.CONF.cachedir)
+        self.assertEqual('/var/lib/os-collect-config', cfg.CONF.cachedir)
         self.assertTrue(extras.safe_hasattr(cfg.CONF, 'ec2'))
         self.assertTrue(extras.safe_hasattr(cfg.CONF, 'cfn'))