From 4696fd4f933b5aec1d7da39ec8fef336d6217734 Mon Sep 17 00:00:00 2001
From: Clint Byrum <clint@fewbar.com>
Date: Thu, 24 Jul 2014 14:37:26 -0700
Subject: [PATCH] Add a local data collector
This collector will collect data from the local system, allowing image
builds or simple processes to influence the metadata.
implements bp tripleo-juno-occ-localdatasource
Change-Id: I0e58e8c631ffe8b63e8b4117df2c9ce2f413044f
---
os_collect_config/collect.py | 9 +-
os_collect_config/exc.py | 4 +
os_collect_config/local.py | 96 +++++++++++++++++
os_collect_config/tests/test_local.py | 142 ++++++++++++++++++++++++++
4 files changed, 250 insertions(+), 1 deletion(-)
create mode 100644 os_collect_config/local.py
create mode 100644 os_collect_config/tests/test_local.py
diff --git a/os_collect_config/collect.py b/os_collect_config/collect.py
index d820f37..a6bec36 100644
--- a/os_collect_config/collect.py
+++ b/os_collect_config/collect.py
@@ -30,6 +30,7 @@ from os_collect_config import exc
from os_collect_config import heat
from os_collect_config import heat_local
from os_collect_config import keystone
+from os_collect_config import local
from os_collect_config import version
from oslo.config import cfg
@@ -81,7 +82,8 @@ logger = log.getLogger('os-collect-config')
COLLECTORS = {ec2.name: ec2,
cfn.name: cfn,
heat.name: heat,
- heat_local.name: heat_local}
+ heat_local.name: heat_local,
+ local.name: local}
def setup_conf():
@@ -94,6 +96,9 @@ def setup_conf():
heat_local_group = cfg.OptGroup(name='heat_local',
title='Heat Local Metadata options')
+ local_group = cfg.OptGroup(name='local',
+ title='Local Metadata options')
+
heat_group = cfg.OptGroup(name='heat',
title='Heat Metadata options')
@@ -103,11 +108,13 @@ def setup_conf():
CONF.register_group(ec2_group)
CONF.register_group(cfn_group)
CONF.register_group(heat_local_group)
+ CONF.register_group(local_group)
CONF.register_group(heat_group)
CONF.register_group(keystone_group)
CONF.register_cli_opts(ec2.opts, group='ec2')
CONF.register_cli_opts(cfn.opts, group='cfn')
CONF.register_cli_opts(heat_local.opts, group='heat_local')
+ CONF.register_cli_opts(local.opts, group='local')
CONF.register_cli_opts(heat.opts, group='heat')
CONF.register_cli_opts(keystone.opts, group='keystone')
diff --git a/os_collect_config/exc.py b/os_collect_config/exc.py
index 88a6485..9522650 100644
--- a/os_collect_config/exc.py
+++ b/os_collect_config/exc.py
@@ -42,5 +42,9 @@ class HeatLocalMetadataNotAvailable(SourceNotAvailable):
"""The local Heat metadata is not available."""
+class LocalMetadataNotAvailable(SourceNotAvailable):
+ """The local metadata is not available."""
+
+
class InvalidArguments(ValueError):
"""Invalid arguments."""
diff --git a/os_collect_config/local.py b/os_collect_config/local.py
new file mode 100644
index 0000000..1fc8d50
--- /dev/null
+++ b/os_collect_config/local.py
@@ -0,0 +1,96 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import locale
+import os
+from oslo.config import cfg
+import stat
+
+from openstack.common import log
+from os_collect_config import exc
+
+LOCAL_DEFAULT_PATHS = ['/var/lib/os-collect-config/local-data']
+CONF = cfg.CONF
+
+opts = [
+ cfg.MultiStrOpt('path',
+ default=LOCAL_DEFAULT_PATHS,
+ help='Local directory to scan for Metadata files.')
+]
+name = 'local'
+logger = log.getLogger(__name__)
+
+
+def _dest_looks_insecure(local_path):
+ '''We allow group writable so owner can let others write.'''
+ looks_insecure = False
+ uid = os.getuid()
+ st = os.stat(local_path)
+ if uid != st[stat.ST_UID]:
+ logger.error('%s is owned by another user. This is a'
+ ' security risk.' % local_path)
+ looks_insecure = True
+ if st.st_mode & stat.S_IWOTH:
+ logger.error('%s is world writable. This is a security risk.'
+ % local_path)
+ looks_insecure = True
+ return looks_insecure
+
+
+class Collector(object):
+ def __init__(self, requests_impl=None):
+ pass
+
+ def collect(self):
+ if len(cfg.CONF.local.path) == 0:
+ raise exc.LocalMetadataNotAvailable
+ final_content = []
+ for local_path in cfg.CONF.local.path:
+ if _dest_looks_insecure(local_path):
+ raise exc.LocalMetadataNotAvailable
+ for data_file in os.listdir(local_path):
+ if data_file.startswith('.'):
+ continue
+ data_file = os.path.join(local_path, data_file)
+ if os.path.isdir(data_file):
+ continue
+ st = os.stat(data_file)
+ if st.st_mode & stat.S_IWOTH:
+ logger.error(
+ '%s is world writable. This is a security risk.' %
+ data_file)
+ raise exc.LocalMetadataNotAvailable
+ with open(data_file) as metadata:
+ try:
+ value = json.loads(metadata.read())
+ except ValueError as e:
+ logger.error(
+ '%s is not valid JSON (%s)' % (data_file, e))
+ raise exc.LocalMetadataNotAvailable
+ basename = os.path.basename(data_file)
+ final_content.append((basename, value))
+ if not final_content:
+ logger.warn('No local metadata found (%s)' %
+ cfg.CONF.local.path)
+
+ # Now sort specifically by C locale
+ def locale_aware_by_first_item(data):
+ return locale.strxfrm(data[0])
+ save_locale = locale.getlocale(locale.LC_ALL)
+ locale.setlocale(locale.LC_ALL, 'C')
+ sorted_content = sorted(final_content, key=locale_aware_by_first_item)
+ locale.setlocale(locale.LC_ALL, save_locale)
+ return sorted_content
diff --git a/os_collect_config/tests/test_local.py b/os_collect_config/tests/test_local.py
new file mode 100644
index 0000000..22f5d16
--- /dev/null
+++ b/os_collect_config/tests/test_local.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import locale
+import os
+import tempfile
+
+import fixtures
+from oslo.config import cfg
+import testtools
+from testtools import matchers
+
+from os_collect_config import collect
+from os_collect_config import exc
+from os_collect_config import local
+
+
+META_DATA = {u'localstrA': u'A',
+ u'localint9': 9,
+ u'localmap_xy': {
+ u'x': 42,
+ u'y': 'foo',
+ }}
+META_DATA2 = {u'localstrA': u'Z',
+ u'localint9': 9}
+
+
+class TestLocal(testtools.TestCase):
+ def setUp(self):
+ super(TestLocal, self).setUp()
+ self.log = self.useFixture(fixtures.FakeLogger())
+ self.useFixture(fixtures.NestedTempfile())
+ self.tdir = tempfile.mkdtemp()
+ collect.setup_conf()
+ self.addCleanup(cfg.CONF.reset)
+ cfg.CONF.register_cli_opts(local.opts, group='local')
+ cfg.CONF.set_override(name='path',
+ override=[self.tdir],
+ group='local')
+
+ def _call_collect(self):
+ md = local.Collector().collect()
+ return md
+
+ def _setup_test_json(self, data, md_base='test.json'):
+ md_name = os.path.join(self.tdir, md_base)
+ with open(md_name, 'w') as md:
+ md.write(json.dumps(data))
+ return md_name
+
+ def test_collect_local(self):
+ self._setup_test_json(META_DATA)
+ local_md = self._call_collect()
+
+ self.assertThat(local_md, matchers.IsInstance(list))
+ self.assertEqual(1, len(local_md))
+ self.assertThat(local_md[0], matchers.IsInstance(tuple))
+ self.assertEqual(2, len(local_md[0]))
+ self.assertEqual('test.json', local_md[0][0])
+
+ only_md = local_md[0][1]
+ self.assertThat(only_md, matchers.IsInstance(dict))
+
+ for k in ('localstrA', 'localint9', 'localmap_xy'):
+ self.assertIn(k, only_md)
+ self.assertEqual(only_md[k], META_DATA[k])
+
+ self.assertEqual('', self.log.output)
+
+ def test_collect_local_world_writable(self):
+ md_name = self._setup_test_json(META_DATA)
+ os.chmod(md_name, 0o666)
+ self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
+ self.assertIn('%s is world writable. This is a security risk.' %
+ md_name, self.log.output)
+
+ def test_collect_local_world_writable_dir(self):
+ self._setup_test_json(META_DATA)
+ os.chmod(self.tdir, 0o666)
+ self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
+ self.assertIn('%s is world writable. This is a security risk.' %
+ self.tdir, self.log.output)
+
+ def test_collect_local_owner_not_uid(self):
+ self._setup_test_json(META_DATA)
+ real_getuid = os.getuid
+
+ def fake_getuid():
+ return real_getuid() + 1
+ self.useFixture(fixtures.MonkeyPatch('os.getuid', fake_getuid))
+ self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
+ self.assertIn('%s is owned by another user. This is a security risk.' %
+ self.tdir, self.log.output)
+
+ def test_collect_local_orders_multiple(self):
+ self._setup_test_json(META_DATA, '00test.json')
+ self._setup_test_json(META_DATA2, '99test.json')
+
+ # Monkey Patch os.listdir so it _always_ returns the wrong sort
+ unpatched_listdir = os.listdir
+
+ def wrong_sort_listdir(path):
+ ret = unpatched_listdir(path)
+ save_locale = locale.getlocale(locale.LC_ALL)
+ locale.setlocale(locale.LC_ALL, 'C')
+ bad_sort = sorted(ret, reverse=True)
+ locale.setlocale(locale.LC_ALL, save_locale)
+ return bad_sort
+ self.useFixture(fixtures.MonkeyPatch('os.listdir', wrong_sort_listdir))
+ local_md = self._call_collect()
+
+ self.assertThat(local_md, matchers.IsInstance(list))
+ self.assertEqual(2, len(local_md))
+ self.assertThat(local_md[0], matchers.IsInstance(tuple))
+
+ self.assertEqual('00test.json', local_md[0][0])
+ md1 = local_md[0][1]
+ self.assertEqual(META_DATA, md1)
+
+ self.assertEqual('99test.json', local_md[1][0])
+ md2 = local_md[1][1]
+ self.assertEqual(META_DATA2, md2)
+
+ def test_collect_invalid_json_fail(self):
+ self._setup_test_json(META_DATA)
+ with open(os.path.join(self.tdir, 'bad.json'), 'w') as badjson:
+ badjson.write('{')
+ self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
+ self.assertIn('is not valid JSON', self.log.output)