openstack/os-collect-config
Code Issues Proposed changes

Merge "Add a local data collector"

Browse Source
This commit is contained in:
Jenkins 2014-08-08 16:59:35 +00:00 committed by Gerrit Code Review
commit 9b28e39fa1
4 changed files with 250 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
os_collect_config/collect.py
View File

@ -30,6 +30,7 @@ from os_collect_config import exc
from os_collect_config import heat
from os_collect_config import heat_local
from os_collect_config import keystone
from os_collect_config import local
from os_collect_config import version
from oslo.config import cfg
@ -81,7 +82,8 @@ logger = log.getLogger('os-collect-config')
COLLECTORS = {ec2.name: ec2,
cfn.name: cfn,
heat.name: heat,
heat_local.name: heat_local}
heat_local.name: heat_local,
local.name: local}
def setup_conf():
@ -94,6 +96,9 @@ def setup_conf():
heat_local_group = cfg.OptGroup(name='heat_local',
title='Heat Local Metadata options')
local_group = cfg.OptGroup(name='local',
title='Local Metadata options')
heat_group = cfg.OptGroup(name='heat',
title='Heat Metadata options')
@ -103,11 +108,13 @@ def setup_conf():
CONF.register_group(ec2_group)
CONF.register_group(cfn_group)
CONF.register_group(heat_local_group)
CONF.register_group(local_group)
CONF.register_group(heat_group)
CONF.register_group(keystone_group)
CONF.register_cli_opts(ec2.opts, group='ec2')
CONF.register_cli_opts(cfn.opts, group='cfn')
CONF.register_cli_opts(heat_local.opts, group='heat_local')
CONF.register_cli_opts(local.opts, group='local')
CONF.register_cli_opts(heat.opts, group='heat')
CONF.register_cli_opts(keystone.opts, group='keystone')

4
os_collect_config/exc.py
View File

@ -46,5 +46,9 @@ class HeatLocalMetadataNotAvailable(SourceNotAvailable):
"""The local Heat metadata is not available."""
class LocalMetadataNotAvailable(SourceNotAvailable):
"""The local metadata is not available."""
class InvalidArguments(ValueError):
"""Invalid arguments."""

96
os_collect_config/local.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import locale
import os
from oslo.config import cfg
import stat
from openstack.common import log
from os_collect_config import exc
LOCAL_DEFAULT_PATHS = ['/var/lib/os-collect-config/local-data']
CONF = cfg.CONF
opts = [
cfg.MultiStrOpt('path',
default=LOCAL_DEFAULT_PATHS,
help='Local directory to scan for Metadata files.')
]
name = 'local'
logger = log.getLogger(__name__)
def _dest_looks_insecure(local_path):
'''We allow group writable so owner can let others write.'''
looks_insecure = False
uid = os.getuid()
st = os.stat(local_path)
if uid != st[stat.ST_UID]:
logger.error('%s is owned by another user. This is a'
' security risk.' % local_path)
looks_insecure = True
if st.st_mode & stat.S_IWOTH:
logger.error('%s is world writable. This is a security risk.'
% local_path)
looks_insecure = True
return looks_insecure
class Collector(object):
def __init__(self, requests_impl=None):
pass
def collect(self):
if len(cfg.CONF.local.path) == 0:
raise exc.LocalMetadataNotAvailable
final_content = []
for local_path in cfg.CONF.local.path:
if _dest_looks_insecure(local_path):
raise exc.LocalMetadataNotAvailable
for data_file in os.listdir(local_path):
if data_file.startswith('.'):
continue
data_file = os.path.join(local_path, data_file)
if os.path.isdir(data_file):
continue
st = os.stat(data_file)
if st.st_mode & stat.S_IWOTH:
logger.error(
'%s is world writable. This is a security risk.' %
data_file)
raise exc.LocalMetadataNotAvailable
with open(data_file) as metadata:
try:
value = json.loads(metadata.read())
except ValueError as e:
logger.error(
'%s is not valid JSON (%s)' % (data_file, e))
raise exc.LocalMetadataNotAvailable
basename = os.path.basename(data_file)
final_content.append((basename, value))
if not final_content:
logger.warn('No local metadata found (%s)' %
cfg.CONF.local.path)
# Now sort specifically by C locale
def locale_aware_by_first_item(data):
return locale.strxfrm(data[0])
save_locale = locale.getlocale(locale.LC_ALL)
locale.setlocale(locale.LC_ALL, 'C')
sorted_content = sorted(final_content, key=locale_aware_by_first_item)
locale.setlocale(locale.LC_ALL, save_locale)
return sorted_content

142
os_collect_config/tests/test_local.py Normal file
View File

@ -0,0 +1,142 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import locale
import os
import tempfile
import fixtures
from oslo.config import cfg
import testtools
from testtools import matchers
from os_collect_config import collect
from os_collect_config import exc
from os_collect_config import local
META_DATA = {u'localstrA': u'A',
u'localint9': 9,
u'localmap_xy': {
u'x': 42,
u'y': 'foo',
}}
META_DATA2 = {u'localstrA': u'Z',
u'localint9': 9}
class TestLocal(testtools.TestCase):
def setUp(self):
super(TestLocal, self).setUp()
self.log = self.useFixture(fixtures.FakeLogger())
self.useFixture(fixtures.NestedTempfile())
self.tdir = tempfile.mkdtemp()
collect.setup_conf()
self.addCleanup(cfg.CONF.reset)
cfg.CONF.register_cli_opts(local.opts, group='local')
cfg.CONF.set_override(name='path',
override=[self.tdir],
group='local')
def _call_collect(self):
md = local.Collector().collect()
return md
def _setup_test_json(self, data, md_base='test.json'):
md_name = os.path.join(self.tdir, md_base)
with open(md_name, 'w') as md:
md.write(json.dumps(data))
return md_name
def test_collect_local(self):
self._setup_test_json(META_DATA)
local_md = self._call_collect()
self.assertThat(local_md, matchers.IsInstance(list))
self.assertEqual(1, len(local_md))
self.assertThat(local_md[0], matchers.IsInstance(tuple))
self.assertEqual(2, len(local_md[0]))
self.assertEqual('test.json', local_md[0][0])
only_md = local_md[0][1]
self.assertThat(only_md, matchers.IsInstance(dict))
for k in ('localstrA', 'localint9', 'localmap_xy'):
self.assertIn(k, only_md)
self.assertEqual(only_md[k], META_DATA[k])
self.assertEqual('', self.log.output)
def test_collect_local_world_writable(self):
md_name = self._setup_test_json(META_DATA)
os.chmod(md_name, 0o666)
self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
self.assertIn('%s is world writable. This is a security risk.' %
md_name, self.log.output)
def test_collect_local_world_writable_dir(self):
self._setup_test_json(META_DATA)
os.chmod(self.tdir, 0o666)
self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
self.assertIn('%s is world writable. This is a security risk.' %
self.tdir, self.log.output)
def test_collect_local_owner_not_uid(self):
self._setup_test_json(META_DATA)
real_getuid = os.getuid
def fake_getuid():
return real_getuid() + 1
self.useFixture(fixtures.MonkeyPatch('os.getuid', fake_getuid))
self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
self.assertIn('%s is owned by another user. This is a security risk.' %
self.tdir, self.log.output)
def test_collect_local_orders_multiple(self):
self._setup_test_json(META_DATA, '00test.json')
self._setup_test_json(META_DATA2, '99test.json')
# Monkey Patch os.listdir so it _always_ returns the wrong sort
unpatched_listdir = os.listdir
def wrong_sort_listdir(path):
ret = unpatched_listdir(path)
save_locale = locale.getlocale(locale.LC_ALL)
locale.setlocale(locale.LC_ALL, 'C')
bad_sort = sorted(ret, reverse=True)
locale.setlocale(locale.LC_ALL, save_locale)
return bad_sort
self.useFixture(fixtures.MonkeyPatch('os.listdir', wrong_sort_listdir))
local_md = self._call_collect()
self.assertThat(local_md, matchers.IsInstance(list))
self.assertEqual(2, len(local_md))
self.assertThat(local_md[0], matchers.IsInstance(tuple))
self.assertEqual('00test.json', local_md[0][0])
md1 = local_md[0][1]
self.assertEqual(META_DATA, md1)
self.assertEqual('99test.json', local_md[1][0])
md2 = local_md[1][1]
self.assertEqual(META_DATA2, md2)
def test_collect_invalid_json_fail(self):
self._setup_test_json(META_DATA)
with open(os.path.join(self.tdir, 'bad.json'), 'w') as badjson:
badjson.write('{')
self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
self.assertIn('is not valid JSON', self.log.output)