Merge "Support control files"
This commit is contained in:
commit
0d39cd5e11
|
@ -22,6 +22,8 @@ import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
from pystache import context
|
from pystache import context
|
||||||
|
import six
|
||||||
|
import yaml
|
||||||
|
|
||||||
from os_apply_config import collect_config
|
from os_apply_config import collect_config
|
||||||
from os_apply_config import config_exception as exc
|
from os_apply_config import config_exception as exc
|
||||||
|
@ -62,6 +64,62 @@ OS_CONFIG_FILES_PATH = os.environ.get(
|
||||||
'OS_CONFIG_FILES_PATH', '/var/lib/os-collect-config/os_config_files.json')
|
'OS_CONFIG_FILES_PATH', '/var/lib/os-collect-config/os_config_files.json')
|
||||||
OS_CONFIG_FILES_PATH_OLD = '/var/run/os-collect-config/os_config_files.json'
|
OS_CONFIG_FILES_PATH_OLD = '/var/run/os-collect-config/os_config_files.json'
|
||||||
|
|
||||||
|
CONTROL_FILE_SUFFIX = ".oac"
|
||||||
|
|
||||||
|
|
||||||
|
class OacFile(object):
|
||||||
|
DEFAULTS = {
|
||||||
|
'allow_empty': True
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(self, body, **kwargs):
|
||||||
|
super(OacFile, self).__init__()
|
||||||
|
self.body = body
|
||||||
|
|
||||||
|
for k, v in six.iteritems(self.DEFAULTS):
|
||||||
|
setattr(self, '_' + k, v)
|
||||||
|
|
||||||
|
for k, v in six.iteritems(kwargs):
|
||||||
|
if not hasattr(self, k):
|
||||||
|
raise exc.ConfigException(
|
||||||
|
"unrecognised file control key '%s'" % (k))
|
||||||
|
setattr(self, k, v)
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
if type(other) is type(self):
|
||||||
|
return self.__dict__ == other.__dict__
|
||||||
|
return False
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
a = ["OacFile(%s" % repr(self.body)]
|
||||||
|
for key, default in six.iteritems(self.DEFAULTS):
|
||||||
|
value = getattr(self, key)
|
||||||
|
if value != default:
|
||||||
|
a.append("%s=%s" % (key, repr(value)))
|
||||||
|
return ", ".join(a) + ")"
|
||||||
|
|
||||||
|
def set(self, key, value):
|
||||||
|
"""Allows setting attrs as an expression rather than a statement."""
|
||||||
|
setattr(self, key, value)
|
||||||
|
return self
|
||||||
|
|
||||||
|
@property
|
||||||
|
def allow_empty(self):
|
||||||
|
"""Returns allow_empty.
|
||||||
|
|
||||||
|
If True and body='', no file will be created and any existing
|
||||||
|
file will be deleted.
|
||||||
|
"""
|
||||||
|
return self._allow_empty
|
||||||
|
|
||||||
|
@allow_empty.setter
|
||||||
|
def allow_empty(self, value):
|
||||||
|
if type(value) is not bool:
|
||||||
|
raise exc.ConfigException(
|
||||||
|
"allow_empty requires Boolean, got: '%s'" % value)
|
||||||
|
self._allow_empty = value
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
def install_config(
|
def install_config(
|
||||||
config_path, template_root, output_path, validate, subhash=None,
|
config_path, template_root, output_path, validate, subhash=None,
|
||||||
|
@ -70,9 +128,9 @@ def install_config(
|
||||||
collect_config.collect_config(config_path, fallback_metadata), subhash)
|
collect_config.collect_config(config_path, fallback_metadata), subhash)
|
||||||
tree = build_tree(template_paths(template_root), config)
|
tree = build_tree(template_paths(template_root), config)
|
||||||
if not validate:
|
if not validate:
|
||||||
for path, contents in tree.items():
|
for path, obj in tree.items():
|
||||||
write_file(os.path.join(
|
write_file(os.path.join(
|
||||||
output_path, strip_prefix('/', path)), contents)
|
output_path, strip_prefix('/', path)), obj)
|
||||||
|
|
||||||
|
|
||||||
def print_key(
|
def print_key(
|
||||||
|
@ -93,7 +151,15 @@ def print_key(
|
||||||
print(str(config))
|
print(str(config))
|
||||||
|
|
||||||
|
|
||||||
def write_file(path, contents):
|
def write_file(path, obj):
|
||||||
|
if not obj.allow_empty and len(obj.body) == 0:
|
||||||
|
if os.path.exists(path):
|
||||||
|
logger.info("deleting %s", path)
|
||||||
|
os.unlink(path)
|
||||||
|
else:
|
||||||
|
logger.info("not creating empty %s", path)
|
||||||
|
return
|
||||||
|
|
||||||
logger.info("writing %s", path)
|
logger.info("writing %s", path)
|
||||||
if os.path.exists(path):
|
if os.path.exists(path):
|
||||||
stat = os.stat(path)
|
stat = os.stat(path)
|
||||||
|
@ -103,20 +169,33 @@ def write_file(path, contents):
|
||||||
d = os.path.dirname(path)
|
d = os.path.dirname(path)
|
||||||
os.path.exists(d) or os.makedirs(d)
|
os.path.exists(d) or os.makedirs(d)
|
||||||
with tempfile.NamedTemporaryFile(dir=d, delete=False) as newfile:
|
with tempfile.NamedTemporaryFile(dir=d, delete=False) as newfile:
|
||||||
if type(contents) == str:
|
if type(obj.body) == str:
|
||||||
contents = contents.encode('utf-8')
|
obj.body = obj.body.encode('utf-8')
|
||||||
newfile.write(contents)
|
newfile.write(obj.body)
|
||||||
os.chmod(newfile.name, mode)
|
os.chmod(newfile.name, mode)
|
||||||
os.chown(newfile.name, uid, gid)
|
os.chown(newfile.name, uid, gid)
|
||||||
os.rename(newfile.name, path)
|
os.rename(newfile.name, path)
|
||||||
|
|
||||||
# return a map of filenames->filecontents
|
|
||||||
|
|
||||||
|
|
||||||
def build_tree(templates, config):
|
def build_tree(templates, config):
|
||||||
|
"""Return a map of filenames to OacFiles."""
|
||||||
res = {}
|
res = {}
|
||||||
for in_file, out_file in templates:
|
for in_file, out_file in templates:
|
||||||
res[out_file] = render_template(in_file, config)
|
try:
|
||||||
|
body = render_template(in_file, config)
|
||||||
|
ctrl_file = in_file + CONTROL_FILE_SUFFIX
|
||||||
|
ctrl_dict = {}
|
||||||
|
if os.path.isfile(ctrl_file):
|
||||||
|
with open(ctrl_file) as cf:
|
||||||
|
ctrl_body = cf.read()
|
||||||
|
ctrl_dict = yaml.safe_load(ctrl_body) or {}
|
||||||
|
if not isinstance(ctrl_dict, dict):
|
||||||
|
raise exc.ConfigException(
|
||||||
|
"header is not a dict: %s" % in_file)
|
||||||
|
res[out_file] = OacFile(body, **ctrl_dict)
|
||||||
|
except exc.ConfigException as e:
|
||||||
|
e.args += in_file,
|
||||||
|
raise
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
@ -163,6 +242,8 @@ def template_paths(root):
|
||||||
res = []
|
res = []
|
||||||
for cur_root, _subdirs, files in os.walk(root):
|
for cur_root, _subdirs, files in os.walk(root):
|
||||||
for f in files:
|
for f in files:
|
||||||
|
if f.endswith(CONTROL_FILE_SUFFIX):
|
||||||
|
continue
|
||||||
inout = (os.path.join(cur_root, f), os.path.join(
|
inout = (os.path.join(cur_root, f), os.path.join(
|
||||||
strip_prefix(root, cur_root), f))
|
strip_prefix(root, cur_root), f))
|
||||||
res.append(inout)
|
res.append(inout)
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
allow_empty: false
|
|
@ -0,0 +1 @@
|
||||||
|
foo
|
|
@ -0,0 +1 @@
|
||||||
|
# comment
|
|
@ -29,7 +29,9 @@ from os_apply_config import config_exception
|
||||||
TEMPLATES = os.path.join(os.path.dirname(__file__), 'templates')
|
TEMPLATES = os.path.join(os.path.dirname(__file__), 'templates')
|
||||||
TEMPLATE_PATHS = [
|
TEMPLATE_PATHS = [
|
||||||
"/etc/glance/script.conf",
|
"/etc/glance/script.conf",
|
||||||
"/etc/keystone/keystone.conf"
|
"/etc/keystone/keystone.conf",
|
||||||
|
"/etc/control/empty",
|
||||||
|
"/etc/control/allow_empty",
|
||||||
]
|
]
|
||||||
|
|
||||||
# config for example tree
|
# config for example tree
|
||||||
|
@ -53,8 +55,14 @@ CONFIG_SUBHASH = {
|
||||||
|
|
||||||
# expected output for example tree
|
# expected output for example tree
|
||||||
OUTPUT = {
|
OUTPUT = {
|
||||||
"/etc/glance/script.conf": "foo\n",
|
"/etc/glance/script.conf": apply_config.OacFile(
|
||||||
"/etc/keystone/keystone.conf": "[foo]\ndatabase = sqlite:///blah\n"
|
"foo\n"),
|
||||||
|
"/etc/keystone/keystone.conf": apply_config.OacFile(
|
||||||
|
"[foo]\ndatabase = sqlite:///blah\n"),
|
||||||
|
"/etc/control/empty": apply_config.OacFile(
|
||||||
|
"foo\n"),
|
||||||
|
"/etc/control/allow_empty": apply_config.OacFile(
|
||||||
|
"").set('allow_empty', False),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -69,6 +77,7 @@ def template(relpath):
|
||||||
|
|
||||||
|
|
||||||
class TestRunOSConfigApplier(testtools.TestCase):
|
class TestRunOSConfigApplier(testtools.TestCase):
|
||||||
|
"""Tests the commandline options."""
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TestRunOSConfigApplier, self).setUp()
|
super(TestRunOSConfigApplier, self).setUp()
|
||||||
|
@ -154,63 +163,78 @@ class OSConfigApplierTestCase(testtools.TestCase):
|
||||||
self.logger = self.useFixture(fixtures.FakeLogger('os-apply-config'))
|
self.logger = self.useFixture(fixtures.FakeLogger('os-apply-config'))
|
||||||
self.useFixture(fixtures.NestedTempfile())
|
self.useFixture(fixtures.NestedTempfile())
|
||||||
|
|
||||||
def test_install_config(self):
|
def write_config(self, config):
|
||||||
fd, path = tempfile.mkstemp()
|
fd, path = tempfile.mkstemp()
|
||||||
with os.fdopen(fd, 'w') as t:
|
with os.fdopen(fd, 'w') as t:
|
||||||
t.write(json.dumps(CONFIG))
|
t.write(json.dumps(config))
|
||||||
t.flush()
|
t.flush()
|
||||||
|
return path
|
||||||
|
|
||||||
|
def check_output_file(self, tmpdir, path, obj):
|
||||||
|
full_path = os.path.join(tmpdir, path[1:])
|
||||||
|
if obj.allow_empty:
|
||||||
|
assert os.path.exists(full_path), "%s doesn't exist" % path
|
||||||
|
self.assertEqual(obj.body, open(full_path).read())
|
||||||
|
else:
|
||||||
|
assert not os.path.exists(full_path), "%s exists" % path
|
||||||
|
|
||||||
|
def test_install_config(self):
|
||||||
|
path = self.write_config(CONFIG)
|
||||||
tmpdir = tempfile.mkdtemp()
|
tmpdir = tempfile.mkdtemp()
|
||||||
apply_config.install_config([path], TEMPLATES, tmpdir, False)
|
apply_config.install_config([path], TEMPLATES, tmpdir, False)
|
||||||
for path, contents in OUTPUT.items():
|
for path, obj in OUTPUT.items():
|
||||||
full_path = os.path.join(tmpdir, path[1:])
|
self.check_output_file(tmpdir, path, obj)
|
||||||
assert os.path.exists(full_path)
|
|
||||||
self.assertEqual(open(full_path).read(), contents)
|
|
||||||
|
|
||||||
def test_install_config_subhash(self):
|
def test_install_config_subhash(self):
|
||||||
fd, tpath = tempfile.mkstemp()
|
tpath = self.write_config(CONFIG_SUBHASH)
|
||||||
with os.fdopen(fd, 'w') as t:
|
|
||||||
t.write(json.dumps(CONFIG_SUBHASH))
|
|
||||||
t.flush()
|
|
||||||
tmpdir = tempfile.mkdtemp()
|
tmpdir = tempfile.mkdtemp()
|
||||||
apply_config.install_config(
|
apply_config.install_config(
|
||||||
[tpath], TEMPLATES, tmpdir, False, 'OpenStack::Config')
|
[tpath], TEMPLATES, tmpdir, False, 'OpenStack::Config')
|
||||||
for path, contents in OUTPUT.items():
|
for path, obj in OUTPUT.items():
|
||||||
full_path = os.path.join(tmpdir, path[1:])
|
self.check_output_file(tmpdir, path, obj)
|
||||||
assert os.path.exists(full_path)
|
|
||||||
self.assertEqual(open(full_path).read(), contents)
|
def test_delete_if_not_allowed_empty(self):
|
||||||
|
path = self.write_config(CONFIG)
|
||||||
|
tmpdir = tempfile.mkdtemp()
|
||||||
|
template = "/etc/control/allow_empty"
|
||||||
|
target_file = os.path.join(tmpdir, template[1:])
|
||||||
|
# Touch the file
|
||||||
|
os.makedirs(os.path.dirname(target_file))
|
||||||
|
open(target_file, 'a').close()
|
||||||
|
apply_config.install_config([path], TEMPLATES, tmpdir, False)
|
||||||
|
# File should be gone
|
||||||
|
self.assertFalse(os.path.exists(target_file))
|
||||||
|
|
||||||
def test_respect_file_permissions(self):
|
def test_respect_file_permissions(self):
|
||||||
fd, path = tempfile.mkstemp()
|
path = self.write_config(CONFIG)
|
||||||
with os.fdopen(fd, 'w') as t:
|
|
||||||
t.write(json.dumps(CONFIG))
|
|
||||||
t.flush()
|
|
||||||
tmpdir = tempfile.mkdtemp()
|
tmpdir = tempfile.mkdtemp()
|
||||||
template = "/etc/keystone/keystone.conf"
|
template = "/etc/keystone/keystone.conf"
|
||||||
target_file = os.path.join(tmpdir, template[1:])
|
target_file = os.path.join(tmpdir, template[1:])
|
||||||
os.makedirs(os.path.dirname(target_file))
|
os.makedirs(os.path.dirname(target_file))
|
||||||
# File doesn't exist, use the default mode (644)
|
# File doesn't exist, use the default mode (644)
|
||||||
apply_config.install_config([path], TEMPLATES, tmpdir, False)
|
apply_config.install_config([path], TEMPLATES, tmpdir, False)
|
||||||
self.assertEqual(os.stat(target_file).st_mode, 0o100644)
|
self.assertEqual(0o100644, os.stat(target_file).st_mode)
|
||||||
self.assertEqual(open(target_file).read(), OUTPUT[template])
|
self.assertEqual(OUTPUT[template].body, open(target_file).read())
|
||||||
# Set a different mode:
|
# Set a different mode:
|
||||||
os.chmod(target_file, 0o600)
|
os.chmod(target_file, 0o600)
|
||||||
apply_config.install_config([path], TEMPLATES, tmpdir, False)
|
apply_config.install_config([path], TEMPLATES, tmpdir, False)
|
||||||
# The permissions should be preserved
|
# The permissions should be preserved
|
||||||
self.assertEqual(os.stat(target_file).st_mode, 0o100600)
|
self.assertEqual(0o100600, os.stat(target_file).st_mode)
|
||||||
self.assertEqual(open(target_file).read(), OUTPUT[template])
|
self.assertEqual(OUTPUT[template].body, open(target_file).read())
|
||||||
|
|
||||||
def test_build_tree(self):
|
def test_build_tree(self):
|
||||||
self.assertEqual(apply_config.build_tree(
|
tree = apply_config.build_tree(
|
||||||
apply_config.template_paths(TEMPLATES), CONFIG), OUTPUT)
|
apply_config.template_paths(TEMPLATES), CONFIG)
|
||||||
|
self.assertEqual(OUTPUT, tree)
|
||||||
|
|
||||||
def test_render_template(self):
|
def test_render_template(self):
|
||||||
# execute executable files, moustache non-executables
|
# execute executable files, moustache non-executables
|
||||||
self.assertEqual(apply_config.render_template(template(
|
self.assertEqual("abc\n", apply_config.render_template(template(
|
||||||
"/etc/glance/script.conf"), {"x": "abc"}), "abc\n")
|
"/etc/glance/script.conf"), {"x": "abc"}))
|
||||||
self.assertRaises(
|
self.assertRaises(
|
||||||
config_exception.ConfigException,
|
config_exception.ConfigException,
|
||||||
apply_config.render_template, template(
|
apply_config.render_template,
|
||||||
"/etc/glance/script.conf"), {})
|
template("/etc/glance/script.conf"), {})
|
||||||
|
|
||||||
def test_render_template_bad_template(self):
|
def test_render_template_bad_template(self):
|
||||||
tdir = self.useFixture(fixtures.TempDir())
|
tdir = self.useFixture(fixtures.TempDir())
|
||||||
|
@ -225,16 +249,17 @@ class OSConfigApplierTestCase(testtools.TestCase):
|
||||||
self.assertIn('Section end tag mismatch', self.logger.output)
|
self.assertIn('Section end tag mismatch', self.logger.output)
|
||||||
|
|
||||||
def test_render_moustache(self):
|
def test_render_moustache(self):
|
||||||
self.assertEqual(apply_config.render_moustache("ab{{x.a}}cd", {
|
self.assertEqual(
|
||||||
"x": {"a": "123"}}), "ab123cd")
|
"ab123cd",
|
||||||
|
apply_config.render_moustache("ab{{x.a}}cd", {"x": {"a": "123"}}))
|
||||||
|
|
||||||
def test_render_moustache_bad_key(self):
|
def test_render_moustache_bad_key(self):
|
||||||
self.assertEqual(apply_config.render_moustache("{{badkey}}", {}), u'')
|
self.assertEqual(u'', apply_config.render_moustache("{{badkey}}", {}))
|
||||||
|
|
||||||
def test_render_executable(self):
|
def test_render_executable(self):
|
||||||
params = {"x": "foo"}
|
params = {"x": "foo"}
|
||||||
self.assertEqual(apply_config.render_executable(template(
|
self.assertEqual("foo\n", apply_config.render_executable(
|
||||||
"/etc/glance/script.conf"), params), "foo\n")
|
template("/etc/glance/script.conf"), params))
|
||||||
|
|
||||||
def test_render_executable_failure(self):
|
def test_render_executable_failure(self):
|
||||||
self.assertRaises(
|
self.assertRaises(
|
||||||
|
@ -247,18 +272,17 @@ class OSConfigApplierTestCase(testtools.TestCase):
|
||||||
actual = apply_config.template_paths(TEMPLATES)
|
actual = apply_config.template_paths(TEMPLATES)
|
||||||
expected.sort(key=lambda tup: tup[1])
|
expected.sort(key=lambda tup: tup[1])
|
||||||
actual.sort(key=lambda tup: tup[1])
|
actual.sort(key=lambda tup: tup[1])
|
||||||
self.assertEqual(actual, expected)
|
self.assertEqual(expected, actual)
|
||||||
|
|
||||||
def test_strip_hash(self):
|
def test_strip_hash(self):
|
||||||
h = {'a': {'b': {'x': 'y'}}, "c": [1, 2, 3]}
|
h = {'a': {'b': {'x': 'y'}}, "c": [1, 2, 3]}
|
||||||
self.assertEqual(apply_config.strip_hash(h, 'a.b'), {'x': 'y'})
|
self.assertEqual({'x': 'y'}, apply_config.strip_hash(h, 'a.b'))
|
||||||
self.assertRaises(config_exception.ConfigException,
|
self.assertRaises(config_exception.ConfigException,
|
||||||
apply_config.strip_hash, h, 'a.nonexistent')
|
apply_config.strip_hash, h, 'a.nonexistent')
|
||||||
self.assertRaises(config_exception.ConfigException,
|
self.assertRaises(config_exception.ConfigException,
|
||||||
apply_config.strip_hash, h, 'a.c')
|
apply_config.strip_hash, h, 'a.c')
|
||||||
|
|
||||||
def test_load_list_from_json(self):
|
def test_load_list_from_json(self):
|
||||||
|
|
||||||
def mkstemp():
|
def mkstemp():
|
||||||
fd, path = tempfile.mkstemp()
|
fd, path = tempfile.mkstemp()
|
||||||
atexit.register(
|
atexit.register(
|
||||||
|
|
|
@ -3,3 +3,5 @@ pbr>=0.6,!=0.7,<1.0
|
||||||
anyjson>=0.3.3
|
anyjson>=0.3.3
|
||||||
argparse
|
argparse
|
||||||
pystache
|
pystache
|
||||||
|
PyYAML>=3.1.0
|
||||||
|
six>=1.7.0
|
||||||
|
|
Loading…
Reference in New Issue