diff --git a/.travis.yml b/.travis.yml index fdeff45..bd7e31d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,8 @@ language: python python: - "2.7" # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install --use-mirrors pystache nose argparse +install: pip install --use-mirrors pystache nose argparse flake8 # # command to run tests, e.g. python setup.py test -script: nosetests +script: ./run_tests.sh notifications: irc: "irc.freenode.org#tripleo" diff --git a/os_config_applier/config_exception.py b/os_config_applier/config_exception.py index 27bb343..722f035 100644 --- a/os_config_applier/config_exception.py +++ b/os_config_applier/config_exception.py @@ -1,2 +1,2 @@ class ConfigException(Exception): - pass + pass diff --git a/os_config_applier/os_config_applier.py b/os_config_applier/os_config_applier.py index 43c5815..a064eb8 100755 --- a/os_config_applier/os_config_applier.py +++ b/os_config_applier/os_config_applier.py @@ -4,101 +4,125 @@ import logging import os import pystache import sys -import tempfile from argparse import ArgumentParser from pystache.context import KeyNotFoundError from subprocess import Popen, PIPE -from value_types import * -from config_exception import * from tempfile import NamedTemporaryFile +from value_types import ensure_type +from config_exception import ConfigException TEMPLATES_DIR = os.environ.get('OS_CONFIG_APPLIER_TEMPLATES', '/opt/stack/os-config-applier/templates') -def install_config(config_path, template_root, output_path, validate, subhash=None): - config = strip_hash( read_config(config_path), subhash) - tree = build_tree( template_paths(template_root), config ) - if not validate: - for path, contents in tree.items(): - write_file( os.path.join(output_path, strip_prefix('/', path)), contents) + +def install_config(config_path, template_root, + output_path, validate, subhash=None): + config = strip_hash(read_config(config_path), subhash) + tree = build_tree(template_paths(template_root), config) + if not validate: + for path, contents in tree.items(): + write_file(os.path.join( + output_path, strip_prefix('/', path)), contents) + def print_key(config_path, key, type_name): - config = read_config(config_path) - keys = key.split('.') - for key in keys: - try: - config = config[key] - except KeyError: - raise KeyError('key %s does not exist in %s' % (key, config_path)) - ensure_type(config, type_name) - print config + config = read_config(config_path) + keys = key.split('.') + for key in keys: + try: + config = config[key] + except KeyError: + raise KeyError('key %s does not exist in %s' % (key, config_path)) + ensure_type(config, type_name) + print config def write_file(path, contents): - logger.info("writing %s", path) - d = os.path.dirname(path) - os.path.exists(d) or os.makedirs(d) - with NamedTemporaryFile(dir=d, delete=False) as newfile: - newfile.write(contents) - os.rename(newfile.name, path) + logger.info("writing %s", path) + d = os.path.dirname(path) + os.path.exists(d) or os.makedirs(d) + with NamedTemporaryFile(dir=d, delete=False) as newfile: + newfile.write(contents) + os.rename(newfile.name, path) # return a map of filenames->filecontents + + def build_tree(templates, config): - res = {} - for in_file, out_file in templates: - res[out_file] = render_template(in_file, config) - return res + res = {} + for in_file, out_file in templates: + res[out_file] = render_template(in_file, config) + return res + def render_template(template, config): - if is_executable(template): - return render_executable(template, config) - else: - try: - return render_moustache(open(template).read(), config) - except KeyNotFoundError as e: - raise ConfigException("key '%s' from template '%s' does not exist in metadata file." % (e.key, template)) - except Exception as e: - raise ConfigException("could not render moustache template %s" % template) + if is_executable(template): + return render_executable(template, config) + else: + try: + return render_moustache(open(template).read(), config) + except KeyNotFoundError as e: + raise ConfigException( + "key '%s' from template '%s' does not exist in metadata file." + % (e.key, template)) + except Exception as e: + raise ConfigException( + "could not render moustache template %s" % template) + def is_executable(path): - return os.path.isfile(path) and os.access(path, os.X_OK) + return os.path.isfile(path) and os.access(path, os.X_OK) + def render_moustache(text, config): - r = pystache.Renderer(missing_tags = 'strict') - return r.render(text, config) + r = pystache.Renderer(missing_tags='strict') + return r.render(text, config) + def render_executable(path, config): - p = Popen([path], stdin=PIPE, stdout=PIPE, stderr=PIPE) - stdout, stderr = p.communicate(json.dumps(config)) - p.wait() - if p.returncode != 0: raise ConfigException("config script failed: %s\n\nwith output:\n\n%s" % (path, stdout + stderr)) - return stdout + p = Popen([path], stdin=PIPE, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate(json.dumps(config)) + p.wait() + if p.returncode != 0: + raise ConfigException( + "config script failed: %s\n\nwith output:\n\n%s" % + (path, stdout + stderr)) + return stdout + def read_config(path): - try: - return json.loads(open(path).read()) - except: - raise ConfigException("invalid metadata file: %s" % path) + try: + return json.loads(open(path).read()) + except: + raise ConfigException("invalid metadata file: %s" % path) + def template_paths(root): - res = [] - for cur_root, subdirs, files in os.walk(root): - for f in files: - inout = ( os.path.join(cur_root, f), os.path.join(strip_prefix(root, cur_root), f) ) - res.append(inout) - return res + res = [] + for cur_root, subdirs, files in os.walk(root): + for f in files: + inout = (os.path.join(cur_root, f), os.path.join( + strip_prefix(root, cur_root), f)) + res.append(inout) + return res + def strip_prefix(prefix, s): - return s[len(prefix):] if s.startswith(prefix) else s + return s[len(prefix):] if s.startswith(prefix) else s + def strip_hash(h, keys): - if not keys: return h - for k in keys.split('.'): - if k in h and isinstance(h[k], dict): - h = h[k] - else: - raise ConfigException("key '%s' does not correspond to a hash in the metadata file" % keys) - return h + if not keys: + return h + for k in keys.split('.'): + if k in h and isinstance(h[k], dict): + h = h[k] + else: + raise ConfigException( + "key '%s' does not correspond to a hash in the metadata file" + % keys) + return h + def parse_opts(argv): parser = ArgumentParser() @@ -107,58 +131,68 @@ def parse_opts(argv): %(default)s)""", default=TEMPLATES_DIR) parser.add_argument('-o', '--output', metavar='OUT_DIR', - help='root directory for output (default: %(default)s)', + help='root directory for output (default:%(default)s)', default='/') parser.add_argument('-m', '--metadata', metavar='METADATA_FILE', help='path to metadata file (default: %(default)s)', default='/var/lib/cloud/data/cfn-init-data') - parser.add_argument('-v', '--validate', help='validate only. do not write files', - default=False, action='store_true') - parser.add_argument('--print-templates', default=False, action='store_true', - help='Print templates root and exit.') + parser.add_argument( + '-v', '--validate', help='validate only. do not write files', + default=False, action='store_true') + parser.add_argument( + '--print-templates', default=False, action='store_true', + help='Print templates root and exit.') parser.add_argument('-s', '--subhash', - help='use the sub-hash named by this key, instead of the full metadata hash') + help='use the sub-hash named by this key,' + ' instead of the full metadata hash') parser.add_argument('--key', metavar='KEY', default=None, - help='print the specified key and exit. (may be used with --type)') + help='print the specified key and exit.' + ' (may be used with --type)') parser.add_argument('--type', default='default', - help='exit with error if the specified --key does not match type. Valid types are ') + help='exit with error if the specified --key does not' + ' match type. Valid types are ') opts = parser.parse_args(argv[1:]) return opts + def main(argv=sys.argv): - opts = parse_opts(argv) - if opts.print_templates: - print(opts.templates) - return 0 + opts = parse_opts(argv) + if opts.print_templates: + print(opts.templates) + return 0 - try: - if opts.templates is None: - raise ConfigException('missing option --templates') + try: + if opts.templates is None: + raise ConfigException('missing option --templates') - if opts.key: - print_key(opts.metadata, opts.key, opts.type) - else: - if not os.access(opts.output, os.W_OK): - raise ConfigException("you don't have permission to write to '%s'" % opts.output) - install_config(opts.metadata, opts.templates, opts.output, - opts.validate, opts.subhash) - logger.info("success") - except ConfigException as e: - logger.error(e) - sys.exit(1) - sys.exit(0) + if opts.key: + print_key(opts.metadata, opts.key, opts.type) + else: + if not os.access(opts.output, os.W_OK): + raise ConfigException( + "you don't have permission to write to '%s'" % opts.output) + install_config(opts.metadata, opts.templates, opts.output, + opts.validate, opts.subhash) + logger.info("success") + except ConfigException as e: + logger.error(e) + sys.exit(1) + sys.exit(0) # logginig LOG_FORMAT = '[%(asctime)s] [%(levelname)s] %(message)s' DATE_FORMAT = '%Y/%m/%d %I:%M:%S %p' + + def add_handler(logger, handler): - handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT)) - logger.addHandler(handler) + handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT)) + logger.addHandler(handler) logger = logging.getLogger('os-config-applier') logger.setLevel(logging.INFO) add_handler(logger, logging.StreamHandler(sys.stdout)) -if os.geteuid() == 0: add_handler(logger, logging.FileHandler('/var/log/os-config-applier.log')) +if os.geteuid() == 0: + add_handler(logger, logging.FileHandler('/var/log/os-config-applier.log')) if __name__ == '__main__': - main(sys.argv) + main(sys.argv) diff --git a/os_config_applier/value_types.py b/os_config_applier/value_types.py index 629b957..7d41fde 100644 --- a/os_config_applier/value_types.py +++ b/os_config_applier/value_types.py @@ -2,14 +2,17 @@ import re from config_exception import ConfigException TYPES = { - "int": "^[0-9]+$", - "default": "^[A-Za-z0-9]+$", - "raw": "." + "int": "^[0-9]+$", + "default": "^[A-Za-z0-9]+$", + "raw": "." } + def ensure_type(string_value, type_name='default'): - if type_name not in TYPES: - raise ValueError("requested validation of unknown type: %s" % type_name) - if not re.match(TYPES[type_name], string_value): - raise ConfigException("cannot interpret value '%s' as type %s" % (string_value, type_name)) - return string_value + if type_name not in TYPES: + raise ValueError( + "requested validation of unknown type: %s" % type_name) + if not re.match(TYPES[type_name], string_value): + raise ConfigException("cannot interpret value '%s' as type %s" % ( + string_value, type_name)) + return string_value diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..123a2b3 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,2 @@ +flake8 --verbose `find . -name '*.py'` +nosetests diff --git a/setup.py b/setup.py index 0955fb2..959c623 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,8 @@ config = { 'install_requires': ['pystache', 'anyjson'], # 'long_description': open('README.md').read(), 'entry_points': { - 'console_scripts': ['os-config-applier = os_config_applier.os_config_applier:main'] + 'console_scripts': [ + 'os-config-applier = os_config_applier.os_config_applier:main'] } } diff --git a/tests/os_config_applier_tests.py b/tests/os_config_applier_tests.py index b384d3d..8efc404 100644 --- a/tests/os_config_applier_tests.py +++ b/tests/os_config_applier_tests.py @@ -1,157 +1,186 @@ import json import os +import sys import subprocess import tempfile from StringIO import StringIO -from nose.tools import * -from os_config_applier.config_exception import * -from os_config_applier.os_config_applier import * +from nose.tools import assert_equal, assert_equals, assert_raises, raises +from os_config_applier.config_exception import ConfigException +from os_config_applier.os_config_applier import ( + main, TEMPLATES_DIR, strip_hash, read_config, template_paths, + render_executable, render_template, render_moustache, install_config, + build_tree) # example template tree TEMPLATES = os.path.join(os.path.dirname(__file__), 'templates') TEMPLATE_PATHS = [ "/etc/glance/script.conf", "/etc/keystone/keystone.conf" - ] +] # config for example tree CONFIG = { - "x": "foo", - "database": { + "x": "foo", + "database": { "url": "sqlite:///blah" - } + } } # config for example tree - with subhash CONFIG_SUBHASH = { - "OpenStack::Config": { + "OpenStack::Config": { "x": "foo", "database": { - "url": "sqlite:///blah" + "url": "sqlite:///blah" + } } - } } # expected output for example tree OUTPUT = { - "/etc/glance/script.conf": "foo\n", - "/etc/keystone/keystone.conf": "[foo]\ndatabase = sqlite:///blah\n" + "/etc/glance/script.conf": "foo\n", + "/etc/keystone/keystone.conf": "[foo]\ndatabase = sqlite:///blah\n" } + def setup(): - pass + pass + def teardown(): - pass + pass + def main_path(): - return os.path.dirname(os.path.realpath(__file__)) + '/../os_config_applier/os_config_applier.py' + return ( + os.path.dirname(os.path.realpath(__file__)) + + '/../os_config_applier/os_config_applier.py') + def template(relpath): - return os.path.join(TEMPLATES, relpath[1:]) + return os.path.join(TEMPLATES, relpath[1:]) + def test_install_config(): - t = tempfile.NamedTemporaryFile() - t.write(json.dumps(CONFIG)) - t.flush() - tmpdir = tempfile.mkdtemp() - install_config(t.name, TEMPLATES, tmpdir, False) - for path, contents in OUTPUT.items(): - full_path = os.path.join(tmpdir, path[1:]) - assert os.path.exists(full_path) - assert_equal( open(full_path).read(), contents ) + t = tempfile.NamedTemporaryFile() + t.write(json.dumps(CONFIG)) + t.flush() + tmpdir = tempfile.mkdtemp() + install_config(t.name, TEMPLATES, tmpdir, False) + for path, contents in OUTPUT.items(): + full_path = os.path.join(tmpdir, path[1:]) + assert os.path.exists(full_path) + assert_equal(open(full_path).read(), contents) + def test_install_config_subhash(): - t = tempfile.NamedTemporaryFile() - t.write(json.dumps(CONFIG_SUBHASH)) - t.flush() - tmpdir = tempfile.mkdtemp() - install_config(t.name, TEMPLATES, tmpdir, False, 'OpenStack::Config') - for path, contents in OUTPUT.items(): - full_path = os.path.join(tmpdir, path[1:]) - assert os.path.exists(full_path) - assert_equal( open(full_path).read(), contents ) + t = tempfile.NamedTemporaryFile() + t.write(json.dumps(CONFIG_SUBHASH)) + t.flush() + tmpdir = tempfile.mkdtemp() + install_config(t.name, TEMPLATES, tmpdir, False, 'OpenStack::Config') + for path, contents in OUTPUT.items(): + full_path = os.path.join(tmpdir, path[1:]) + assert os.path.exists(full_path) + assert_equal(open(full_path).read(), contents) + def test_print_key(): - t = tempfile.NamedTemporaryFile() - t.write(json.dumps(CONFIG)) - t.flush() - out = subprocess.check_output([main_path(), '--metadata', t.name, '--key', - 'database.url', '--type', 'raw'], - stderr=subprocess.STDOUT) - assert_equals(CONFIG['database']['url'], out.rstrip()) + t = tempfile.NamedTemporaryFile() + t.write(json.dumps(CONFIG)) + t.flush() + out = subprocess.check_output([main_path(), '--metadata', t.name, '--key', + 'database.url', '--type', 'raw'], + stderr=subprocess.STDOUT) + assert_equals(CONFIG['database']['url'], out.rstrip()) + @raises(subprocess.CalledProcessError) def test_print_key_missing(): - t = tempfile.NamedTemporaryFile() - t.write(json.dumps(CONFIG)) - t.flush() - out = subprocess.check_output([main_path(), '--metadata', t.name, '--key', - 'does.not.exist'], stderr=subprocess.STDOUT) + t = tempfile.NamedTemporaryFile() + t.write(json.dumps(CONFIG)) + t.flush() + subprocess.check_output([main_path(), '--metadata', t.name, '--key', + 'does.not.exist'], stderr=subprocess.STDOUT) + @raises(subprocess.CalledProcessError) def test_print_key_wrong_type(): - t = tempfile.NamedTemporaryFile() - t.write(json.dumps(CONFIG)) - t.flush() - out = subprocess.check_output([main_path(), '--metadata', t.name, '--key', - 'x', '--type', 'int'], - stderr=subprocess.STDOUT) + t = tempfile.NamedTemporaryFile() + t.write(json.dumps(CONFIG)) + t.flush() + subprocess.check_output([main_path(), '--metadata', t.name, '--key', + 'x', '--type', 'int'], stderr=subprocess.STDOUT) def test_build_tree(): - assert_equals( build_tree(template_paths(TEMPLATES), CONFIG), OUTPUT ) + assert_equals(build_tree(template_paths(TEMPLATES), CONFIG), OUTPUT) + def test_render_template(): - # execute executable files, moustache non-executables - assert render_template(template("/etc/glance/script.conf"), {"x": "abc"}) == "abc\n" - assert_raises(ConfigException, render_template, template("/etc/glance/script.conf"), {}) + # execute executable files, moustache non-executables + assert render_template(template( + "/etc/glance/script.conf"), {"x": "abc"}) == "abc\n" + assert_raises(ConfigException, render_template, template( + "/etc/glance/script.conf"), {}) + def test_render_moustache(): - assert_equals( render_moustache("ab{{x.a}}cd", {"x": {"a": "123"}}), "ab123cd" ) + assert_equals(render_moustache("ab{{x.a}}cd", { + "x": {"a": "123"}}), "ab123cd") + @raises(Exception) def test_render_moustache_bad_key(): - render_moustache("{{badkey}}", {}) + render_moustache("{{badkey}}", {}) + def test_render_executable(): - params = {"x": "foo"} - assert render_executable(template("/etc/glance/script.conf"), params) == "foo\n" + params = {"x": "foo"} + assert render_executable(template( + "/etc/glance/script.conf"), params) == "foo\n" + @raises(ConfigException) def test_render_executable_failure(): - render_executable(template("/etc/glance/script.conf"), {}) + render_executable(template("/etc/glance/script.conf"), {}) + def test_template_paths(): - expected = map(lambda p: (template(p), p), TEMPLATE_PATHS) - actual = template_paths(TEMPLATES) - expected.sort(key=lambda tup: tup[1]) - actual.sort(key=lambda tup: tup[1]) - assert_equals( actual , expected) + expected = map(lambda p: (template(p), p), TEMPLATE_PATHS) + actual = template_paths(TEMPLATES) + expected.sort(key=lambda tup: tup[1]) + actual.sort(key=lambda tup: tup[1]) + assert_equals(actual, expected) + def test_read_config(): - with tempfile.NamedTemporaryFile() as t: - d = {"a": {"b": ["c", "d"] } } - t.write(json.dumps(d)) - t.flush() - assert_equals( read_config(t.name), d ) + with tempfile.NamedTemporaryFile() as t: + d = {"a": {"b": ["c", "d"]}} + t.write(json.dumps(d)) + t.flush() + assert_equals(read_config(t.name), d) + @raises(ConfigException) def test_read_config_bad_json(): - with tempfile.NamedTemporaryFile() as t: - t.write("{{{{") - t.flush() - read_config(t.name) + with tempfile.NamedTemporaryFile() as t: + t.write("{{{{") + t.flush() + read_config(t.name) + @raises(Exception) def test_read_config_no_file(): - read_config("/nosuchfile") + read_config("/nosuchfile") + def test_strip_hash(): - h = {'a': {'b': {'x': 'y'} }, "c": [1, 2, 3] } - assert_equals( strip_hash(h, 'a.b'), {'x': 'y'}) - assert_raises(ConfigException, strip_hash, h, 'a.nonexistent') - assert_raises(ConfigException, strip_hash, h, 'a.c') + h = {'a': {'b': {'x': 'y'}}, "c": [1, 2, 3]} + assert_equals(strip_hash(h, 'a.b'), {'x': 'y'}) + assert_raises(ConfigException, strip_hash, h, 'a.nonexistent') + assert_raises(ConfigException, strip_hash, h, 'a.c') + def test_print_templates(): save_stdout = sys.stdout diff --git a/tests/value_type_tests.py b/tests/value_type_tests.py index c9d6032..61772a8 100644 --- a/tests/value_type_tests.py +++ b/tests/value_type_tests.py @@ -1,18 +1,21 @@ -from nose.tools import * -from os_config_applier.os_config_applier import * -from os_config_applier.config_exception import * -from os_config_applier.value_types import * +from nose.tools import assert_equals, raises +from os_config_applier.config_exception import ConfigException +from os_config_applier.value_types import ensure_type + @raises(ValueError) def test_unknown_type(): - ensure_type("foo", "badtype") + ensure_type("foo", "badtype") + def test_int(): - assert_equals("123", ensure_type("123", "int")) + assert_equals("123", ensure_type("123", "int")) + def test_defualt(): - assert_equals("foobar", ensure_type("foobar", "default")) + assert_equals("foobar", ensure_type("foobar", "default")) + @raises(ConfigException) def test_default_bad(): - ensure_type("foo\nbar", "default") + ensure_type("foo\nbar", "default")