From a13845a737e08e635a95910ce6ce0d4a96b39511 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Tue, 2 Jul 2013 10:17:13 -0700 Subject: [PATCH] Run all collectors. Also refactor internal API a bit to make code more generic. --- os_collect_config/cfn.py | 3 ++- os_collect_config/collect.py | 34 +++++++++++++++++++------ os_collect_config/ec2.py | 3 ++- os_collect_config/tests/test_cfn.py | 14 +++++----- os_collect_config/tests/test_collect.py | 26 +++++++++++++++++-- os_collect_config/tests/test_ec2.py | 4 +-- 6 files changed, 63 insertions(+), 21 deletions(-) diff --git a/os_collect_config/cfn.py b/os_collect_config/cfn.py index fc129f1..01a4451 100644 --- a/os_collect_config/cfn.py +++ b/os_collect_config/cfn.py @@ -32,9 +32,10 @@ opts = [ cfg.MultiStrOpt('path', help='Path to Metadata'), ] +name = 'cfn' -class CollectCfn(object): +class Collector(object): def __init__(self, requests_impl=common.requests): self._requests_impl = requests_impl self._session = requests_impl.Session() diff --git a/os_collect_config/collect.py b/os_collect_config/collect.py index 9b01243..b89f664 100644 --- a/os_collect_config/collect.py +++ b/os_collect_config/collect.py @@ -36,6 +36,8 @@ opts = [ CONF = cfg.CONF logger = log.getLogger('os-collect-config') +COLLECTORS = (ec2, cfn) + def setup_conf(): ec2_group = cfg.OptGroup(name='ec2', @@ -52,24 +54,40 @@ def setup_conf(): CONF.register_cli_opts(opts) -def __main__(ec2_requests=common.requests): +def __main__(requests_impl_map=None): setup_conf() CONF(prog="os-collect-config") log.setup("os-collect-config") - ec2_content = ec2.CollectEc2(requests_impl=ec2_requests).collect() + + final_content = {} + paths = [] + for collector in COLLECTORS: + if requests_impl_map and collector.name in requests_impl_map: + requests_impl = requests_impl_map[collector.name] + else: + requests_impl = common.requests + content = collector.Collector(requests_impl=requests_impl).collect() + + any_changed = False + if CONF.command: + (changed, path) = cache.store(collector.name, content) + any_changed |= changed + paths.append(path) + else: + final_content[collector.name] = content if CONF.command: - (changed, ec2_path) = cache.store('ec2', ec2_content) - if changed: - paths = [ec2_path] + if any_changed: env = dict(os.environ) env["OS_CONFIG_FILES"] = ':'.join(paths) logger.info("Executing %s" % CONF.command) subprocess.call(CONF.command, env=env, shell=True) - cache.commit('ec2') + for collector in COLLECTORS: + cache.commit(collector.name) + else: + logger.debug("No changes detected.") else: - content = {'ec2': ec2_content} - print json.dumps(content, indent=1) + print json.dumps(final_content, indent=1) if __name__ == '__main__': diff --git a/os_collect_config/ec2.py b/os_collect_config/ec2.py index c355fdd..1c12b27 100644 --- a/os_collect_config/ec2.py +++ b/os_collect_config/ec2.py @@ -27,9 +27,10 @@ opts = [ default=EC2_METADATA_URL, help='URL to query for EC2 Metadata') ] +name = 'ec2' -class CollectEc2(object): +class Collector(object): def __init__(self, requests_impl=common.requests): self._requests_impl = requests_impl self.session = requests_impl.Session() diff --git a/os_collect_config/tests/test_cfn.py b/os_collect_config/tests/test_cfn.py index 66f5598..0aa466d 100644 --- a/os_collect_config/tests/test_cfn.py +++ b/os_collect_config/tests/test_cfn.py @@ -85,7 +85,7 @@ class TestCfn(testtools.TestCase): cfg.CONF.cfn.path = ['foo.Metadata'] def test_collect_cfn(self): - cfn_md = cfn.CollectCfn(requests_impl=FakeRequests(self)).collect() + cfn_md = cfn.Collector(requests_impl=FakeRequests(self)).collect() self.assertThat(cfn_md, matchers.IsInstance(dict)) for k in ('int1', 'strfoo', 'map_ab'): @@ -95,37 +95,37 @@ class TestCfn(testtools.TestCase): self.assertEquals('', self.log.output) def test_collect_cfn_fail(self): - cfn_collect = cfn.CollectCfn(requests_impl=FakeFailRequests) + cfn_collect = cfn.Collector(requests_impl=FakeFailRequests) self.assertRaises(exc.CfnMetadataNotAvailable, cfn_collect.collect) self.assertIn('Forbidden', self.log.output) def test_collect_cfn_no_path(self): cfg.CONF.cfn.path = None - cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self)) + cfn_collect = cfn.Collector(requests_impl=FakeRequests(self)) self.assertRaises(exc.CfnMetadataNotConfigured, cfn_collect.collect) self.assertIn('No path configured', self.log.output) def test_collect_cfn_bad_path(self): cfg.CONF.cfn.path = ['foo'] - cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self)) + cfn_collect = cfn.Collector(requests_impl=FakeRequests(self)) self.assertRaises(exc.CfnMetadataNotConfigured, cfn_collect.collect) self.assertIn('Path not in format', self.log.output) def test_collect_cfn_no_metadata_url(self): cfg.CONF.cfn.metadata_url = None - cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self)) + cfn_collect = cfn.Collector(requests_impl=FakeRequests(self)) self.assertRaises(exc.CfnMetadataNotConfigured, cfn_collect.collect) self.assertIn('No metadata_url configured', self.log.output) def test_collect_cfn_missing_sub_path(self): cfg.CONF.cfn.path = ['foo.Metadata.not_there'] - cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self)) + cfn_collect = cfn.Collector(requests_impl=FakeRequests(self)) self.assertRaises(exc.CfnMetadataNotConfigured, cfn_collect.collect) self.assertIn('Sub-path could not be found', self.log.output) def test_collect_cfn_sub_path(self): cfg.CONF.cfn.path = ['foo.Metadata.map_ab'] - cfn_collect = cfn.CollectCfn(requests_impl=FakeRequests(self)) + cfn_collect = cfn.Collector(requests_impl=FakeRequests(self)) content = cfn_collect.collect() self.assertThat(content, matchers.IsInstance(dict)) self.assertIn(u'b', content) diff --git a/os_collect_config/tests/test_collect.py b/os_collect_config/tests/test_collect.py index 1d7779d..5625ad3 100644 --- a/os_collect_config/tests/test_collect.py +++ b/os_collect_config/tests/test_collect.py @@ -22,6 +22,7 @@ import testtools from testtools import matchers from os_collect_config import collect +from os_collect_config.tests import test_cfn from os_collect_config.tests import test_ec2 @@ -41,6 +42,12 @@ class TestCollect(testtools.TestCase): cache_dir.path, '--config-file', '/dev/null', + '--cfn-metadata-url', + 'http://127.0.0.1:8000/', + '--cfn-stack-name', + 'foo', + '--cfn-path', + 'foo.Metadata' ] self.useFixture( fixtures.MonkeyPatch('sys.argv', fake_args)) @@ -57,12 +64,18 @@ class TestCollect(testtools.TestCase): with open(path) as cfg_file: contents = json.loads(cfg_file.read()) keys_found.update(set(contents.keys())) + # From test_ec2.FakeRequests self.assertIn("local-ipv4", keys_found) self.assertIn("reservation-id", keys_found) + # From test_cfn.FakeRequests + self.assertIn("int1", keys_found) + self.assertIn("map_ab", keys_found) self.useFixture(fixtures.MonkeyPatch('subprocess.call', fake_call)) - collect.__main__(ec2_requests=test_ec2.FakeRequests) + requests_impl_map = {'ec2': test_ec2.FakeRequests, + 'cfn': test_cfn.FakeRequests(self)} + collect.__main__(requests_impl_map=requests_impl_map) self.assertTrue(self.called_fake_call) @@ -71,16 +84,25 @@ class TestCollect(testtools.TestCase): 'os-collect-config', '--config-file', '/dev/null', + '--cfn-metadata-url', + 'http://127.0.0.1:8000/', + '--cfn-stack-name', + 'foo', + '--cfn-path', + 'foo.Metadata' ] self.useFixture( fixtures.MonkeyPatch('sys.argv', fake_args)) output = self.useFixture(fixtures.ByteStream('stdout')) self.useFixture( fixtures.MonkeyPatch('sys.stdout', output.stream)) - collect.__main__(ec2_requests=test_ec2.FakeRequests) + requests_impl_map = {'ec2': test_ec2.FakeRequests, + 'cfn': test_cfn.FakeRequests(self)} + collect.__main__(requests_impl_map=requests_impl_map) out_struct = json.loads(output.stream.getvalue()) self.assertThat(out_struct, matchers.IsInstance(dict)) self.assertIn('ec2', out_struct) + self.assertIn('cfn', out_struct) class TestConf(testtools.TestCase): diff --git a/os_collect_config/tests/test_ec2.py b/os_collect_config/tests/test_ec2.py index ac8f938..ba023c4 100644 --- a/os_collect_config/tests/test_ec2.py +++ b/os_collect_config/tests/test_ec2.py @@ -91,7 +91,7 @@ class TestEc2(testtools.TestCase): def test_collect_ec2(self): collect.setup_conf() - ec2_md = ec2.CollectEc2(requests_impl=FakeRequests).collect() + ec2_md = ec2.Collector(requests_impl=FakeRequests).collect() self.assertThat(ec2_md, matchers.IsInstance(dict)) for k in ('public-ipv4', 'instance-id', 'hostname'): @@ -108,6 +108,6 @@ class TestEc2(testtools.TestCase): def test_collect_ec2_fail(self): collect.setup_conf() - collect_ec2 = ec2.CollectEc2(requests_impl=FakeFailRequests) + collect_ec2 = ec2.Collector(requests_impl=FakeFailRequests) self.assertRaises(exc.Ec2MetadataNotAvailable, collect_ec2.collect) self.assertIn('Forbidden', self.log.output)