Merge "Add a local data collector"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
9b28e39fa1
|
|
@ -30,6 +30,7 @@ from os_collect_config import exc
|
|||
from os_collect_config import heat
|
||||
from os_collect_config import heat_local
|
||||
from os_collect_config import keystone
|
||||
from os_collect_config import local
|
||||
from os_collect_config import version
|
||||
from oslo.config import cfg
|
||||
|
||||
|
|
@ -81,7 +82,8 @@ logger = log.getLogger('os-collect-config')
|
|||
COLLECTORS = {ec2.name: ec2,
|
||||
cfn.name: cfn,
|
||||
heat.name: heat,
|
||||
heat_local.name: heat_local}
|
||||
heat_local.name: heat_local,
|
||||
local.name: local}
|
||||
|
||||
|
||||
def setup_conf():
|
||||
|
|
@ -94,6 +96,9 @@ def setup_conf():
|
|||
heat_local_group = cfg.OptGroup(name='heat_local',
|
||||
title='Heat Local Metadata options')
|
||||
|
||||
local_group = cfg.OptGroup(name='local',
|
||||
title='Local Metadata options')
|
||||
|
||||
heat_group = cfg.OptGroup(name='heat',
|
||||
title='Heat Metadata options')
|
||||
|
||||
|
|
@ -103,11 +108,13 @@ def setup_conf():
|
|||
CONF.register_group(ec2_group)
|
||||
CONF.register_group(cfn_group)
|
||||
CONF.register_group(heat_local_group)
|
||||
CONF.register_group(local_group)
|
||||
CONF.register_group(heat_group)
|
||||
CONF.register_group(keystone_group)
|
||||
CONF.register_cli_opts(ec2.opts, group='ec2')
|
||||
CONF.register_cli_opts(cfn.opts, group='cfn')
|
||||
CONF.register_cli_opts(heat_local.opts, group='heat_local')
|
||||
CONF.register_cli_opts(local.opts, group='local')
|
||||
CONF.register_cli_opts(heat.opts, group='heat')
|
||||
CONF.register_cli_opts(keystone.opts, group='keystone')
|
||||
|
||||
|
|
|
|||
|
|
@ -46,5 +46,9 @@ class HeatLocalMetadataNotAvailable(SourceNotAvailable):
|
|||
"""The local Heat metadata is not available."""
|
||||
|
||||
|
||||
class LocalMetadataNotAvailable(SourceNotAvailable):
|
||||
"""The local metadata is not available."""
|
||||
|
||||
|
||||
class InvalidArguments(ValueError):
|
||||
"""Invalid arguments."""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import locale
|
||||
import os
|
||||
from oslo.config import cfg
|
||||
import stat
|
||||
|
||||
from openstack.common import log
|
||||
from os_collect_config import exc
|
||||
|
||||
LOCAL_DEFAULT_PATHS = ['/var/lib/os-collect-config/local-data']
|
||||
CONF = cfg.CONF
|
||||
|
||||
opts = [
|
||||
cfg.MultiStrOpt('path',
|
||||
default=LOCAL_DEFAULT_PATHS,
|
||||
help='Local directory to scan for Metadata files.')
|
||||
]
|
||||
name = 'local'
|
||||
logger = log.getLogger(__name__)
|
||||
|
||||
|
||||
def _dest_looks_insecure(local_path):
|
||||
'''We allow group writable so owner can let others write.'''
|
||||
looks_insecure = False
|
||||
uid = os.getuid()
|
||||
st = os.stat(local_path)
|
||||
if uid != st[stat.ST_UID]:
|
||||
logger.error('%s is owned by another user. This is a'
|
||||
' security risk.' % local_path)
|
||||
looks_insecure = True
|
||||
if st.st_mode & stat.S_IWOTH:
|
||||
logger.error('%s is world writable. This is a security risk.'
|
||||
% local_path)
|
||||
looks_insecure = True
|
||||
return looks_insecure
|
||||
|
||||
|
||||
class Collector(object):
|
||||
def __init__(self, requests_impl=None):
|
||||
pass
|
||||
|
||||
def collect(self):
|
||||
if len(cfg.CONF.local.path) == 0:
|
||||
raise exc.LocalMetadataNotAvailable
|
||||
final_content = []
|
||||
for local_path in cfg.CONF.local.path:
|
||||
if _dest_looks_insecure(local_path):
|
||||
raise exc.LocalMetadataNotAvailable
|
||||
for data_file in os.listdir(local_path):
|
||||
if data_file.startswith('.'):
|
||||
continue
|
||||
data_file = os.path.join(local_path, data_file)
|
||||
if os.path.isdir(data_file):
|
||||
continue
|
||||
st = os.stat(data_file)
|
||||
if st.st_mode & stat.S_IWOTH:
|
||||
logger.error(
|
||||
'%s is world writable. This is a security risk.' %
|
||||
data_file)
|
||||
raise exc.LocalMetadataNotAvailable
|
||||
with open(data_file) as metadata:
|
||||
try:
|
||||
value = json.loads(metadata.read())
|
||||
except ValueError as e:
|
||||
logger.error(
|
||||
'%s is not valid JSON (%s)' % (data_file, e))
|
||||
raise exc.LocalMetadataNotAvailable
|
||||
basename = os.path.basename(data_file)
|
||||
final_content.append((basename, value))
|
||||
if not final_content:
|
||||
logger.warn('No local metadata found (%s)' %
|
||||
cfg.CONF.local.path)
|
||||
|
||||
# Now sort specifically by C locale
|
||||
def locale_aware_by_first_item(data):
|
||||
return locale.strxfrm(data[0])
|
||||
save_locale = locale.getlocale(locale.LC_ALL)
|
||||
locale.setlocale(locale.LC_ALL, 'C')
|
||||
sorted_content = sorted(final_content, key=locale_aware_by_first_item)
|
||||
locale.setlocale(locale.LC_ALL, save_locale)
|
||||
return sorted_content
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import locale
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import fixtures
|
||||
from oslo.config import cfg
|
||||
import testtools
|
||||
from testtools import matchers
|
||||
|
||||
from os_collect_config import collect
|
||||
from os_collect_config import exc
|
||||
from os_collect_config import local
|
||||
|
||||
|
||||
META_DATA = {u'localstrA': u'A',
|
||||
u'localint9': 9,
|
||||
u'localmap_xy': {
|
||||
u'x': 42,
|
||||
u'y': 'foo',
|
||||
}}
|
||||
META_DATA2 = {u'localstrA': u'Z',
|
||||
u'localint9': 9}
|
||||
|
||||
|
||||
class TestLocal(testtools.TestCase):
|
||||
def setUp(self):
|
||||
super(TestLocal, self).setUp()
|
||||
self.log = self.useFixture(fixtures.FakeLogger())
|
||||
self.useFixture(fixtures.NestedTempfile())
|
||||
self.tdir = tempfile.mkdtemp()
|
||||
collect.setup_conf()
|
||||
self.addCleanup(cfg.CONF.reset)
|
||||
cfg.CONF.register_cli_opts(local.opts, group='local')
|
||||
cfg.CONF.set_override(name='path',
|
||||
override=[self.tdir],
|
||||
group='local')
|
||||
|
||||
def _call_collect(self):
|
||||
md = local.Collector().collect()
|
||||
return md
|
||||
|
||||
def _setup_test_json(self, data, md_base='test.json'):
|
||||
md_name = os.path.join(self.tdir, md_base)
|
||||
with open(md_name, 'w') as md:
|
||||
md.write(json.dumps(data))
|
||||
return md_name
|
||||
|
||||
def test_collect_local(self):
|
||||
self._setup_test_json(META_DATA)
|
||||
local_md = self._call_collect()
|
||||
|
||||
self.assertThat(local_md, matchers.IsInstance(list))
|
||||
self.assertEqual(1, len(local_md))
|
||||
self.assertThat(local_md[0], matchers.IsInstance(tuple))
|
||||
self.assertEqual(2, len(local_md[0]))
|
||||
self.assertEqual('test.json', local_md[0][0])
|
||||
|
||||
only_md = local_md[0][1]
|
||||
self.assertThat(only_md, matchers.IsInstance(dict))
|
||||
|
||||
for k in ('localstrA', 'localint9', 'localmap_xy'):
|
||||
self.assertIn(k, only_md)
|
||||
self.assertEqual(only_md[k], META_DATA[k])
|
||||
|
||||
self.assertEqual('', self.log.output)
|
||||
|
||||
def test_collect_local_world_writable(self):
|
||||
md_name = self._setup_test_json(META_DATA)
|
||||
os.chmod(md_name, 0o666)
|
||||
self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
|
||||
self.assertIn('%s is world writable. This is a security risk.' %
|
||||
md_name, self.log.output)
|
||||
|
||||
def test_collect_local_world_writable_dir(self):
|
||||
self._setup_test_json(META_DATA)
|
||||
os.chmod(self.tdir, 0o666)
|
||||
self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
|
||||
self.assertIn('%s is world writable. This is a security risk.' %
|
||||
self.tdir, self.log.output)
|
||||
|
||||
def test_collect_local_owner_not_uid(self):
|
||||
self._setup_test_json(META_DATA)
|
||||
real_getuid = os.getuid
|
||||
|
||||
def fake_getuid():
|
||||
return real_getuid() + 1
|
||||
self.useFixture(fixtures.MonkeyPatch('os.getuid', fake_getuid))
|
||||
self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
|
||||
self.assertIn('%s is owned by another user. This is a security risk.' %
|
||||
self.tdir, self.log.output)
|
||||
|
||||
def test_collect_local_orders_multiple(self):
|
||||
self._setup_test_json(META_DATA, '00test.json')
|
||||
self._setup_test_json(META_DATA2, '99test.json')
|
||||
|
||||
# Monkey Patch os.listdir so it _always_ returns the wrong sort
|
||||
unpatched_listdir = os.listdir
|
||||
|
||||
def wrong_sort_listdir(path):
|
||||
ret = unpatched_listdir(path)
|
||||
save_locale = locale.getlocale(locale.LC_ALL)
|
||||
locale.setlocale(locale.LC_ALL, 'C')
|
||||
bad_sort = sorted(ret, reverse=True)
|
||||
locale.setlocale(locale.LC_ALL, save_locale)
|
||||
return bad_sort
|
||||
self.useFixture(fixtures.MonkeyPatch('os.listdir', wrong_sort_listdir))
|
||||
local_md = self._call_collect()
|
||||
|
||||
self.assertThat(local_md, matchers.IsInstance(list))
|
||||
self.assertEqual(2, len(local_md))
|
||||
self.assertThat(local_md[0], matchers.IsInstance(tuple))
|
||||
|
||||
self.assertEqual('00test.json', local_md[0][0])
|
||||
md1 = local_md[0][1]
|
||||
self.assertEqual(META_DATA, md1)
|
||||
|
||||
self.assertEqual('99test.json', local_md[1][0])
|
||||
md2 = local_md[1][1]
|
||||
self.assertEqual(META_DATA2, md2)
|
||||
|
||||
def test_collect_invalid_json_fail(self):
|
||||
self._setup_test_json(META_DATA)
|
||||
with open(os.path.join(self.tdir, 'bad.json'), 'w') as badjson:
|
||||
badjson.write('{')
|
||||
self.assertRaises(exc.LocalMetadataNotAvailable, self._call_collect)
|
||||
self.assertIn('is not valid JSON', self.log.output)
|
||||
Loading…
Reference in New Issue