Run all collectors.

Also refactor internal API a bit to make code more generic.

os_collect_config/cfn.py

@@ -32,9 +32,10 @@ opts = [
     cfg.MultiStrOpt('path',
                     help='Path to Metadata'),
 ]
+name = 'cfn'
 
 
-class CollectCfn(object):
+class Collector(object):
     def __init__(self, requests_impl=common.requests):
         self._requests_impl = requests_impl
         self._session = requests_impl.Session()

os_collect_config/collect.py

@@ -36,6 +36,8 @@ opts = [
 CONF = cfg.CONF
 logger = log.getLogger('os-collect-config')
 
+COLLECTORS = (ec2, cfn)
+
 
 def setup_conf():
     ec2_group = cfg.OptGroup(name='ec2',
@@ -52,24 +54,40 @@ def setup_conf():
     CONF.register_cli_opts(opts)
 
 
-def __main__(ec2_requests=common.requests):
+def __main__(requests_impl_map=None):
     setup_conf()
     CONF(prog="os-collect-config")
     log.setup("os-collect-config")
-    ec2_content = ec2.CollectEc2(requests_impl=ec2_requests).collect()
+
+    final_content = {}
+    paths = []
+    for collector in COLLECTORS:
+        if requests_impl_map and collector.name in requests_impl_map:
+            requests_impl = requests_impl_map[collector.name]
+        else:
+            requests_impl = common.requests
+        content = collector.Collector(requests_impl=requests_impl).collect()
+
+        any_changed = False
+        if CONF.command:
+            (changed, path) = cache.store(collector.name, content)
+            any_changed |= changed
+            paths.append(path)
+        else:
+            final_content[collector.name] = content
 
     if CONF.command:
-        (changed, ec2_path) = cache.store('ec2', ec2_content)
+        if any_changed:
-        if changed:
-            paths = [ec2_path]
             env = dict(os.environ)
             env["OS_CONFIG_FILES"] = ':'.join(paths)
             logger.info("Executing %s" % CONF.command)
             subprocess.call(CONF.command, env=env, shell=True)
-            cache.commit('ec2')
+            for collector in COLLECTORS:
+                cache.commit(collector.name)
+        else:
+            logger.debug("No changes detected.")
     else:
-        content = {'ec2': ec2_content}
+        print json.dumps(final_content, indent=1)
-        print json.dumps(content, indent=1)
 
 
 if __name__ == '__main__':

os_collect_config/ec2.py

@@ -27,9 +27,10 @@ opts = [
                default=EC2_METADATA_URL,
                help='URL to query for EC2 Metadata')
 ]
+name = 'ec2'
 
 
-class CollectEc2(object):
+class Collector(object):
     def __init__(self, requests_impl=common.requests):
         self._requests_impl = requests_impl
         self.session = requests_impl.Session()

os_collect_config/tests/test_cfn.py

@@ -85,7 +85,7 @@ class TestCfn(testtools.TestCase):
         cfg.CONF.cfn.path = ['foo.Metadata']
 
     def test_collect_cfn(self):
-        cfn_md = cfn.CollectCfn(requests_impl=FakeRequests(self)).collect()
+        cfn_md = cfn.Collector(requests_impl=FakeRequests(self)).collect()
         self.assertThat(cfn_md, matchers.IsInstance(dict))
 
         for k in ('int1', 'strfoo', 'map_ab'):
@@ -95,37 +95,37 @@ class TestCfn(testtools.TestCase):
         self.assertEquals('', self.log.output)
 
     def test_collect_cfn_fail(self):
-        cfn_collect = cfn.CollectCfn(requests_impl=FakeFailRequests)
+        cfn_collect = cfn.Collector(requests_impl=FakeFailRequests)
         self.assertRaises(exc.CfnMetadataNotAvailable, cfn_collect.collect)
         self.assertIn('Forbidden', self.log.output)
 
     def test_collect_cfn_no_path(self):
         cfg.CONF.cfn.path = None
-        cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self))
+        cfn_collect = cfn.Collector(requests_impl=FakeRequests(self))
         self.assertRaises(exc.CfnMetadataNotConfigured, cfn_collect.collect)
         self.assertIn('No path configured', self.log.output)
 
     def test_collect_cfn_bad_path(self):
         cfg.CONF.cfn.path = ['foo']
-        cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self))
+        cfn_collect = cfn.Collector(requests_impl=FakeRequests(self))
         self.assertRaises(exc.CfnMetadataNotConfigured, cfn_collect.collect)
         self.assertIn('Path not in format', self.log.output)
 
     def test_collect_cfn_no_metadata_url(self):
         cfg.CONF.cfn.metadata_url = None
-        cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self))
+        cfn_collect = cfn.Collector(requests_impl=FakeRequests(self))
         self.assertRaises(exc.CfnMetadataNotConfigured, cfn_collect.collect)
         self.assertIn('No metadata_url configured', self.log.output)
 
     def test_collect_cfn_missing_sub_path(self):
         cfg.CONF.cfn.path = ['foo.Metadata.not_there']
-        cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self))
+        cfn_collect = cfn.Collector(requests_impl=FakeRequests(self))
         self.assertRaises(exc.CfnMetadataNotConfigured, cfn_collect.collect)
         self.assertIn('Sub-path could not be found', self.log.output)
 
     def test_collect_cfn_sub_path(self):
         cfg.CONF.cfn.path = ['foo.Metadata.map_ab']
-        cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self))
+        cfn_collect = cfn.Collector(requests_impl=FakeRequests(self))
         content = cfn_collect.collect()
         self.assertThat(content, matchers.IsInstance(dict))
         self.assertIn(u'b', content)

os_collect_config/tests/test_collect.py

@@ -22,6 +22,7 @@ import testtools
 from testtools import matchers
 
 from os_collect_config import collect
+from os_collect_config.tests import test_cfn
 from os_collect_config.tests import test_ec2
 
 
@@ -41,6 +42,12 @@ class TestCollect(testtools.TestCase):
             cache_dir.path,
             '--config-file',
             '/dev/null',
+            '--cfn-metadata-url',
+            'http://127.0.0.1:8000/',
+            '--cfn-stack-name',
+            'foo',
+            '--cfn-path',
+            'foo.Metadata'
         ]
         self.useFixture(
             fixtures.MonkeyPatch('sys.argv', fake_args))
@@ -57,12 +64,18 @@ class TestCollect(testtools.TestCase):
                 with open(path) as cfg_file:
                     contents = json.loads(cfg_file.read())
                     keys_found.update(set(contents.keys()))
+            # From test_ec2.FakeRequests
             self.assertIn("local-ipv4", keys_found)
             self.assertIn("reservation-id", keys_found)
+            # From test_cfn.FakeRequests
+            self.assertIn("int1", keys_found)
+            self.assertIn("map_ab", keys_found)
 
         self.useFixture(fixtures.MonkeyPatch('subprocess.call', fake_call))
 
-        collect.__main__(ec2_requests=test_ec2.FakeRequests)
+        requests_impl_map = {'ec2': test_ec2.FakeRequests,
+                             'cfn': test_cfn.FakeRequests(self)}
+        collect.__main__(requests_impl_map=requests_impl_map)
 
         self.assertTrue(self.called_fake_call)
 
@@ -71,16 +84,25 @@ class TestCollect(testtools.TestCase):
             'os-collect-config',
             '--config-file',
             '/dev/null',
+            '--cfn-metadata-url',
+            'http://127.0.0.1:8000/',
+            '--cfn-stack-name',
+            'foo',
+            '--cfn-path',
+            'foo.Metadata'
         ]
         self.useFixture(
             fixtures.MonkeyPatch('sys.argv', fake_args))
         output = self.useFixture(fixtures.ByteStream('stdout'))
         self.useFixture(
             fixtures.MonkeyPatch('sys.stdout', output.stream))
-        collect.__main__(ec2_requests=test_ec2.FakeRequests)
+        requests_impl_map = {'ec2': test_ec2.FakeRequests,
+                             'cfn': test_cfn.FakeRequests(self)}
+        collect.__main__(requests_impl_map=requests_impl_map)
         out_struct = json.loads(output.stream.getvalue())
         self.assertThat(out_struct, matchers.IsInstance(dict))
         self.assertIn('ec2', out_struct)
+        self.assertIn('cfn', out_struct)
 
 
 class TestConf(testtools.TestCase):

os_collect_config/tests/test_ec2.py

@@ -91,7 +91,7 @@ class TestEc2(testtools.TestCase):
 
     def test_collect_ec2(self):
         collect.setup_conf()
-        ec2_md = ec2.CollectEc2(requests_impl=FakeRequests).collect()
+        ec2_md = ec2.Collector(requests_impl=FakeRequests).collect()
         self.assertThat(ec2_md, matchers.IsInstance(dict))
 
         for k in ('public-ipv4', 'instance-id', 'hostname'):
@@ -108,6 +108,6 @@ class TestEc2(testtools.TestCase):
 
     def test_collect_ec2_fail(self):
         collect.setup_conf()
-        collect_ec2 = ec2.CollectEc2(requests_impl=FakeFailRequests)
+        collect_ec2 = ec2.Collector(requests_impl=FakeFailRequests)
         self.assertRaises(exc.Ec2MetadataNotAvailable, collect_ec2.collect)
         self.assertIn('Forbidden', self.log.output)