openstack/os-apply-config
Code Issues Proposed changes

Merge "Support control files"

Browse Source
This commit is contained in:
Jenkins 2014-09-15 06:45:39 +00:00 committed by Gerrit Code Review
commit 0d39cd5e11
7 changed files with 158 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
os_apply_config/apply_config.py
View File

@ -22,6 +22,8 @@ import sys
import tempfile
from pystache import context
import six
import yaml
from os_apply_config import collect_config
from os_apply_config import config_exception as exc
@ -62,6 +64,62 @@ OS_CONFIG_FILES_PATH = os.environ.get(
'OS_CONFIG_FILES_PATH', '/var/lib/os-collect-config/os_config_files.json')
OS_CONFIG_FILES_PATH_OLD = '/var/run/os-collect-config/os_config_files.json'
CONTROL_FILE_SUFFIX = ".oac"
class OacFile(object):
DEFAULTS = {
'allow_empty': True
}
def __init__(self, body, **kwargs):
super(OacFile, self).__init__()
self.body = body
for k, v in six.iteritems(self.DEFAULTS):
setattr(self, '_' + k, v)
for k, v in six.iteritems(kwargs):
if not hasattr(self, k):
raise exc.ConfigException(
"unrecognised file control key '%s'" % (k))
setattr(self, k, v)
def __eq__(self, other):
if type(other) is type(self):
return self.__dict__ == other.__dict__
return False
def __repr__(self):
a = ["OacFile(%s" % repr(self.body)]
for key, default in six.iteritems(self.DEFAULTS):
value = getattr(self, key)
if value != default:
a.append("%s=%s" % (key, repr(value)))
return ", ".join(a) + ")"
def set(self, key, value):
"""Allows setting attrs as an expression rather than a statement."""
setattr(self, key, value)
return self
@property
def allow_empty(self):
"""Returns allow_empty.
If True and body='', no file will be created and any existing
file will be deleted.
"""
return self._allow_empty
@allow_empty.setter
def allow_empty(self, value):
if type(value) is not bool:
raise exc.ConfigException(
"allow_empty requires Boolean, got: '%s'" % value)
self._allow_empty = value
return self
def install_config(
config_path, template_root, output_path, validate, subhash=None,
@ -70,9 +128,9 @@ def install_config(
collect_config.collect_config(config_path, fallback_metadata), subhash)
tree = build_tree(template_paths(template_root), config)
if not validate:
for path, contents in tree.items():
for path, obj in tree.items():
write_file(os.path.join(
output_path, strip_prefix('/', path)), contents)
output_path, strip_prefix('/', path)), obj)
def print_key(
@ -93,7 +151,15 @@ def print_key(
print(str(config))
def write_file(path, contents):
def write_file(path, obj):
if not obj.allow_empty and len(obj.body) == 0:
if os.path.exists(path):
logger.info("deleting %s", path)
os.unlink(path)
else:
logger.info("not creating empty %s", path)
return
logger.info("writing %s", path)
if os.path.exists(path):
stat = os.stat(path)
@ -103,20 +169,33 @@ def write_file(path, contents):
d = os.path.dirname(path)
os.path.exists(d) or os.makedirs(d)
with tempfile.NamedTemporaryFile(dir=d, delete=False) as newfile:
if type(contents) == str:
contents = contents.encode('utf-8')
newfile.write(contents)
if type(obj.body) == str:
obj.body = obj.body.encode('utf-8')
newfile.write(obj.body)
os.chmod(newfile.name, mode)
os.chown(newfile.name, uid, gid)
os.rename(newfile.name, path)
# return a map of filenames->filecontents
def build_tree(templates, config):
"""Return a map of filenames to OacFiles."""
res = {}
for in_file, out_file in templates:
res[out_file] = render_template(in_file, config)
try:
body = render_template(in_file, config)
ctrl_file = in_file + CONTROL_FILE_SUFFIX
ctrl_dict = {}
if os.path.isfile(ctrl_file):
with open(ctrl_file) as cf:
ctrl_body = cf.read()
ctrl_dict = yaml.safe_load(ctrl_body) or {}
if not isinstance(ctrl_dict, dict):
raise exc.ConfigException(
"header is not a dict: %s" % in_file)
res[out_file] = OacFile(body, **ctrl_dict)
except exc.ConfigException as e:
e.args += in_file,
raise
return res
@ -163,6 +242,8 @@ def template_paths(root):
res = []
for cur_root, _subdirs, files in os.walk(root):
for f in files:
if f.endswith(CONTROL_FILE_SUFFIX):
continue
inout = (os.path.join(cur_root, f), os.path.join(
strip_prefix(root, cur_root), f))
res.append(inout)

0
os_apply_config/tests/templates/etc/control/allow_empty Normal file
View File

1
os_apply_config/tests/templates/etc/control/allow_empty.oac Normal file
View File

@ -0,0 +1 @@
allow_empty: false

1
os_apply_config/tests/templates/etc/control/empty Normal file
View File

@ -0,0 +1 @@
foo

1
os_apply_config/tests/templates/etc/control/empty.oac Normal file
View File

@ -0,0 +1 @@
# comment

102
os_apply_config/tests/test_apply_config.py
View File

@ -29,7 +29,9 @@ from os_apply_config import config_exception
TEMPLATES = os.path.join(os.path.dirname(__file__), 'templates')
TEMPLATE_PATHS = [
"/etc/glance/script.conf",
"/etc/keystone/keystone.conf"
"/etc/keystone/keystone.conf",
"/etc/control/empty",
"/etc/control/allow_empty",
]
# config for example tree
@ -53,8 +55,14 @@ CONFIG_SUBHASH = {
# expected output for example tree
OUTPUT = {
"/etc/glance/script.conf": "foo\n",
"/etc/keystone/keystone.conf": "[foo]\ndatabase = sqlite:///blah\n"
"/etc/glance/script.conf": apply_config.OacFile(
"foo\n"),
"/etc/keystone/keystone.conf": apply_config.OacFile(
"[foo]\ndatabase = sqlite:///blah\n"),
"/etc/control/empty": apply_config.OacFile(
"foo\n"),
"/etc/control/allow_empty": apply_config.OacFile(
"").set('allow_empty', False),
}
@ -69,6 +77,7 @@ def template(relpath):
class TestRunOSConfigApplier(testtools.TestCase):
"""Tests the commandline options."""
def setUp(self):
super(TestRunOSConfigApplier, self).setUp()
@ -154,63 +163,78 @@ class OSConfigApplierTestCase(testtools.TestCase):
self.logger = self.useFixture(fixtures.FakeLogger('os-apply-config'))
self.useFixture(fixtures.NestedTempfile())
def test_install_config(self):
def write_config(self, config):
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'w') as t:
t.write(json.dumps(CONFIG))
t.write(json.dumps(config))
t.flush()
return path
def check_output_file(self, tmpdir, path, obj):
full_path = os.path.join(tmpdir, path[1:])
if obj.allow_empty:
assert os.path.exists(full_path), "%s doesn't exist" % path
self.assertEqual(obj.body, open(full_path).read())
else:
assert not os.path.exists(full_path), "%s exists" % path
def test_install_config(self):
path = self.write_config(CONFIG)
tmpdir = tempfile.mkdtemp()
apply_config.install_config([path], TEMPLATES, tmpdir, False)
for path, contents in OUTPUT.items():
full_path = os.path.join(tmpdir, path[1:])
assert os.path.exists(full_path)
self.assertEqual(open(full_path).read(), contents)
for path, obj in OUTPUT.items():
self.check_output_file(tmpdir, path, obj)
def test_install_config_subhash(self):
fd, tpath = tempfile.mkstemp()
with os.fdopen(fd, 'w') as t:
t.write(json.dumps(CONFIG_SUBHASH))
t.flush()
tpath = self.write_config(CONFIG_SUBHASH)
tmpdir = tempfile.mkdtemp()
apply_config.install_config(
[tpath], TEMPLATES, tmpdir, False, 'OpenStack::Config')
for path, contents in OUTPUT.items():
full_path = os.path.join(tmpdir, path[1:])
assert os.path.exists(full_path)
self.assertEqual(open(full_path).read(), contents)
for path, obj in OUTPUT.items():
self.check_output_file(tmpdir, path, obj)
def test_delete_if_not_allowed_empty(self):
path = self.write_config(CONFIG)
tmpdir = tempfile.mkdtemp()
template = "/etc/control/allow_empty"
target_file = os.path.join(tmpdir, template[1:])
# Touch the file
os.makedirs(os.path.dirname(target_file))
open(target_file, 'a').close()
apply_config.install_config([path], TEMPLATES, tmpdir, False)
# File should be gone
self.assertFalse(os.path.exists(target_file))
def test_respect_file_permissions(self):
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'w') as t:
t.write(json.dumps(CONFIG))
t.flush()
path = self.write_config(CONFIG)
tmpdir = tempfile.mkdtemp()
template = "/etc/keystone/keystone.conf"
target_file = os.path.join(tmpdir, template[1:])
os.makedirs(os.path.dirname(target_file))
# File doesn't exist, use the default mode (644)
apply_config.install_config([path], TEMPLATES, tmpdir, False)
self.assertEqual(os.stat(target_file).st_mode, 0o100644)
self.assertEqual(open(target_file).read(), OUTPUT[template])
self.assertEqual(0o100644, os.stat(target_file).st_mode)
self.assertEqual(OUTPUT[template].body, open(target_file).read())
# Set a different mode:
os.chmod(target_file, 0o600)
apply_config.install_config([path], TEMPLATES, tmpdir, False)
# The permissions should be preserved
self.assertEqual(os.stat(target_file).st_mode, 0o100600)
self.assertEqual(open(target_file).read(), OUTPUT[template])
self.assertEqual(0o100600, os.stat(target_file).st_mode)
self.assertEqual(OUTPUT[template].body, open(target_file).read())
def test_build_tree(self):
self.assertEqual(apply_config.build_tree(
apply_config.template_paths(TEMPLATES), CONFIG), OUTPUT)
tree = apply_config.build_tree(
apply_config.template_paths(TEMPLATES), CONFIG)
self.assertEqual(OUTPUT, tree)
def test_render_template(self):
# execute executable files, moustache non-executables
self.assertEqual(apply_config.render_template(template(
"/etc/glance/script.conf"), {"x": "abc"}), "abc\n")
self.assertEqual("abc\n", apply_config.render_template(template(
"/etc/glance/script.conf"), {"x": "abc"}))
self.assertRaises(
config_exception.ConfigException,
apply_config.render_template, template(
"/etc/glance/script.conf"), {})
apply_config.render_template,
template("/etc/glance/script.conf"), {})
def test_render_template_bad_template(self):
tdir = self.useFixture(fixtures.TempDir())
@ -225,16 +249,17 @@ class OSConfigApplierTestCase(testtools.TestCase):
self.assertIn('Section end tag mismatch', self.logger.output)
def test_render_moustache(self):
self.assertEqual(apply_config.render_moustache("ab{{x.a}}cd", {
"x": {"a": "123"}}), "ab123cd")
self.assertEqual(
"ab123cd",
apply_config.render_moustache("ab{{x.a}}cd", {"x": {"a": "123"}}))
def test_render_moustache_bad_key(self):
self.assertEqual(apply_config.render_moustache("{{badkey}}", {}), u'')
self.assertEqual(u'', apply_config.render_moustache("{{badkey}}", {}))
def test_render_executable(self):
params = {"x": "foo"}
self.assertEqual(apply_config.render_executable(template(
"/etc/glance/script.conf"), params), "foo\n")
self.assertEqual("foo\n", apply_config.render_executable(
template("/etc/glance/script.conf"), params))
def test_render_executable_failure(self):
self.assertRaises(
@ -247,18 +272,17 @@ class OSConfigApplierTestCase(testtools.TestCase):
actual = apply_config.template_paths(TEMPLATES)
expected.sort(key=lambda tup: tup[1])
actual.sort(key=lambda tup: tup[1])
self.assertEqual(actual, expected)
self.assertEqual(expected, actual)
def test_strip_hash(self):
h = {'a': {'b': {'x': 'y'}}, "c": [1, 2, 3]}
self.assertEqual(apply_config.strip_hash(h, 'a.b'), {'x': 'y'})
self.assertEqual({'x': 'y'}, apply_config.strip_hash(h, 'a.b'))
self.assertRaises(config_exception.ConfigException,
apply_config.strip_hash, h, 'a.nonexistent')
self.assertRaises(config_exception.ConfigException,
apply_config.strip_hash, h, 'a.c')
def test_load_list_from_json(self):
def mkstemp():
fd, path = tempfile.mkstemp()
atexit.register(

2
requirements.txt
View File

@ -3,3 +3,5 @@ pbr>=0.6,!=0.7,<1.0
anyjson>=0.3.3
argparse
pystache
PyYAML>=3.1.0
six>=1.7.0