388 lines
14 KiB
Python
Executable File
388 lines
14 KiB
Python
Executable File
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
from pystache import context
|
|
import yaml
|
|
|
|
from os_apply_config import collect_config
|
|
from os_apply_config import config_exception as exc
|
|
from os_apply_config import oac_file
|
|
from os_apply_config import renderers
|
|
from os_apply_config import value_types
|
|
from os_apply_config import version
|
|
|
|
DEFAULT_TEMPLATES_DIR = '/usr/libexec/os-apply-config/templates'
|
|
|
|
|
|
def templates_dir():
|
|
"""Determine the default templates directory path
|
|
|
|
If the OS_CONFIG_APPLIER_TEMPLATES environment variable has been set,
|
|
use its value.
|
|
Otherwise, select a default path based on which directories exist on the
|
|
system, preferring the newer paths but still allowing the old ones for
|
|
backwards compatibility.
|
|
"""
|
|
templates_dir = os.environ.get('OS_CONFIG_APPLIER_TEMPLATES', None)
|
|
if templates_dir is None:
|
|
templates_dir = '/opt/stack/os-apply-config/templates'
|
|
if not os.path.isdir(templates_dir):
|
|
# Backwards compat with the old name.
|
|
templates_dir = '/opt/stack/os-config-applier/templates'
|
|
if (os.path.isdir(templates_dir) and
|
|
not os.path.isdir(DEFAULT_TEMPLATES_DIR)):
|
|
logging.warning('Template directory %s is deprecated. The '
|
|
'recommended location for template files is %s',
|
|
templates_dir, DEFAULT_TEMPLATES_DIR)
|
|
else:
|
|
templates_dir = DEFAULT_TEMPLATES_DIR
|
|
return templates_dir
|
|
|
|
|
|
TEMPLATES_DIR = templates_dir()
|
|
OS_CONFIG_FILES_PATH = os.environ.get(
|
|
'OS_CONFIG_FILES_PATH', '/var/lib/os-collect-config/os_config_files.json')
|
|
OS_CONFIG_FILES_PATH_OLD = '/var/run/os-collect-config/os_config_files.json'
|
|
|
|
CONTROL_FILE_SUFFIX = ".oac"
|
|
|
|
|
|
def install_config(
|
|
config_path, template_root, output_path, validate, subhash=None,
|
|
fallback_metadata=None):
|
|
config = strip_hash(
|
|
collect_config.collect_config(config_path, fallback_metadata), subhash)
|
|
tree = build_tree(template_paths(template_root), config)
|
|
if not validate:
|
|
for path, obj in tree.items():
|
|
write_file(os.path.join(
|
|
output_path, strip_prefix('/', path)), obj)
|
|
|
|
|
|
def _extract_key(config_path, key, fallback_metadata=None):
|
|
config = collect_config.collect_config(config_path, fallback_metadata)
|
|
keys = key.split('.')
|
|
for key in keys:
|
|
try:
|
|
config = config[key]
|
|
if config is None:
|
|
raise TypeError()
|
|
except (KeyError, TypeError):
|
|
try:
|
|
if type(config) == list:
|
|
config = config[int(key)]
|
|
continue
|
|
except (IndexError, ValueError):
|
|
pass
|
|
return None
|
|
return config
|
|
|
|
|
|
def print_key(
|
|
config_path, key, type_name, default=None, fallback_metadata=None):
|
|
config = collect_config.collect_config(config_path, fallback_metadata)
|
|
config = _extract_key(config_path, key, fallback_metadata)
|
|
if config is None:
|
|
if default is not None:
|
|
print(str(default))
|
|
return
|
|
else:
|
|
raise exc.ConfigException(
|
|
'key %s does not exist in %s' % (key, config_path))
|
|
value_types.ensure_type(str(config), type_name)
|
|
if isinstance(config, (dict, list, bool)):
|
|
print(json.dumps(config))
|
|
else:
|
|
print(str(config))
|
|
|
|
|
|
def boolean_key(metadata, key, fallback_metadata):
|
|
config = _extract_key(metadata, key, fallback_metadata)
|
|
if not isinstance(config, bool):
|
|
return -1
|
|
if config:
|
|
return 0
|
|
else:
|
|
return 1
|
|
|
|
|
|
def write_file(path, obj):
|
|
if not obj.allow_empty and len(obj.body) == 0:
|
|
if os.path.exists(path):
|
|
logger.info("deleting %s", path)
|
|
os.unlink(path)
|
|
else:
|
|
logger.info("not creating empty %s", path)
|
|
return
|
|
|
|
logger.info("writing %s", path)
|
|
if os.path.exists(path):
|
|
stat = os.stat(path)
|
|
mode, uid, gid = stat.st_mode, stat.st_uid, stat.st_gid
|
|
else:
|
|
mode, uid, gid = 0o644, -1, -1
|
|
mode = obj.mode or mode
|
|
if obj.owner is not None:
|
|
uid = obj.owner
|
|
if obj.group is not None:
|
|
gid = obj.group
|
|
|
|
d = os.path.dirname(path)
|
|
os.path.exists(d) or os.makedirs(d)
|
|
with tempfile.NamedTemporaryFile(dir=d, delete=False) as newfile:
|
|
if type(obj.body) == str:
|
|
obj.body = obj.body.encode('utf-8')
|
|
newfile.write(obj.body)
|
|
os.chmod(newfile.name, mode)
|
|
os.chown(newfile.name, uid, gid)
|
|
os.rename(newfile.name, path)
|
|
|
|
|
|
def build_tree(templates, config):
|
|
"""Return a map of filenames to OacFiles."""
|
|
res = {}
|
|
for in_file, out_file in templates:
|
|
try:
|
|
body = render_template(in_file, config)
|
|
ctrl_file = in_file + CONTROL_FILE_SUFFIX
|
|
ctrl_dict = {}
|
|
if os.path.isfile(ctrl_file):
|
|
with open(ctrl_file) as cf:
|
|
ctrl_body = cf.read()
|
|
ctrl_dict = yaml.safe_load(ctrl_body) or {}
|
|
if not isinstance(ctrl_dict, dict):
|
|
raise exc.ConfigException(
|
|
"header is not a dict: %s" % in_file)
|
|
res[out_file] = oac_file.OacFile(body, **ctrl_dict)
|
|
except exc.ConfigException as e:
|
|
e.args += in_file,
|
|
raise
|
|
return res
|
|
|
|
|
|
def render_template(template, config):
|
|
if is_executable(template):
|
|
return render_executable(template, config)
|
|
else:
|
|
try:
|
|
return render_moustache(open(template).read(), config)
|
|
except context.KeyNotFoundError as e:
|
|
raise exc.ConfigException(
|
|
"key '%s' from template '%s' does not exist in metadata file."
|
|
% (e.key, template))
|
|
except Exception as e:
|
|
logger.error("%s", e)
|
|
raise exc.ConfigException(
|
|
"could not render moustache template %s" % template)
|
|
|
|
|
|
def is_executable(path):
|
|
return os.path.isfile(path) and os.access(path, os.X_OK)
|
|
|
|
|
|
def render_moustache(text, config):
|
|
r = renderers.JsonRenderer(missing_tags='ignore')
|
|
return r.render(text, config)
|
|
|
|
|
|
def render_executable(path, config):
|
|
p = subprocess.Popen([path],
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
stdout, stderr = p.communicate(json.dumps(config).encode('utf-8'))
|
|
p.wait()
|
|
if p.returncode != 0:
|
|
raise exc.ConfigException(
|
|
"config script failed: %s\n\nwith output:\n\n%s" %
|
|
(path, stdout + stderr))
|
|
return stdout.decode('utf-8')
|
|
|
|
|
|
def template_paths(root):
|
|
res = []
|
|
for cur_root, _subdirs, files in os.walk(root):
|
|
for f in files:
|
|
if f.endswith(CONTROL_FILE_SUFFIX):
|
|
continue
|
|
inout = (os.path.join(cur_root, f), os.path.join(
|
|
strip_prefix(root, cur_root), f))
|
|
res.append(inout)
|
|
return res
|
|
|
|
|
|
def strip_prefix(prefix, s):
|
|
return s[len(prefix):] if s.startswith(prefix) else s
|
|
|
|
|
|
def strip_hash(h, keys):
|
|
if not keys:
|
|
return h
|
|
for k in keys.split('.'):
|
|
if k in h and isinstance(h[k], dict):
|
|
h = h[k]
|
|
else:
|
|
raise exc.ConfigException(
|
|
"key '%s' does not correspond to a hash in the metadata file"
|
|
% keys)
|
|
return h
|
|
|
|
|
|
def parse_opts(argv):
|
|
parser = argparse.ArgumentParser(
|
|
description='Reads and merges JSON configuration files specified'
|
|
' by colon separated environment variable OS_CONFIG_FILES, unless'
|
|
' overridden by command line option --metadata. If no files are'
|
|
' specified this way, falls back to legacy behavior of searching'
|
|
' the fallback metadata path for a single config file.')
|
|
parser.add_argument('-t', '--templates', metavar='TEMPLATE_ROOT',
|
|
help="""path to template root directory (default:
|
|
%(default)s)""",
|
|
default=TEMPLATES_DIR)
|
|
parser.add_argument('-o', '--output', metavar='OUT_DIR',
|
|
help='root directory for output (default:%(default)s)',
|
|
default='/')
|
|
parser.add_argument('-m', '--metadata', metavar='METADATA_FILE', nargs='*',
|
|
help='Overrides environment variable OS_CONFIG_FILES.'
|
|
' Specify multiple times, rather than separate files'
|
|
' with ":".',
|
|
default=[])
|
|
parser.add_argument('--fallback-metadata', metavar='FALLBACK_METADATA',
|
|
nargs='*', help='Files to search when OS_CONFIG_FILES'
|
|
' is empty. (default: %(default)s)',
|
|
default=['/var/cache/heat-cfntools/last_metadata',
|
|
'/var/lib/heat-cfntools/cfn-init-data',
|
|
'/var/lib/cloud/data/cfn-init-data'])
|
|
parser.add_argument(
|
|
'-v', '--validate', help='validate only. do not write files',
|
|
default=False, action='store_true')
|
|
parser.add_argument(
|
|
'--print-templates', default=False, action='store_true',
|
|
help='Print templates root and exit.')
|
|
parser.add_argument('-s', '--subhash',
|
|
help='use the sub-hash named by this key,'
|
|
' instead of the full metadata hash')
|
|
parser.add_argument('--key', metavar='KEY', default=None,
|
|
help='print the specified key and exit.'
|
|
' (may be used with --type and --key-default)')
|
|
parser.add_argument('--type', default='default',
|
|
help='exit with error if the specified --key does not'
|
|
' match type. Valid types are'
|
|
' <int|default|netaddress|netdevice|dsn|'
|
|
'swiftdevices|raw>')
|
|
parser.add_argument('--key-default',
|
|
help='This option only affects running with --key.'
|
|
' Print this if key is not found. This value is'
|
|
' not subject to type restrictions. If --key is'
|
|
' specified and no default is specified, program'
|
|
' exits with an error on missing key.')
|
|
parser.add_argument('--boolean-key',
|
|
help='This option is incompatible with --key.'
|
|
' Use this to evaluate whether a value is'
|
|
' boolean true or false. The return code of the'
|
|
' command will be 0 for true, 1 for false, and -1'
|
|
' for non-boolean values.')
|
|
parser.add_argument('--version', action='version',
|
|
version=version.version_info.version_string())
|
|
parser.add_argument('--os-config-files',
|
|
default=OS_CONFIG_FILES_PATH,
|
|
help='Set path to os_config_files.json')
|
|
opts = parser.parse_args(argv[1:])
|
|
|
|
return opts
|
|
|
|
|
|
def load_list_from_json(json_file):
|
|
json_obj = []
|
|
if os.path.exists(json_file):
|
|
with open(json_file) as ocf:
|
|
json_obj = json.loads(ocf.read())
|
|
if not isinstance(json_obj, list):
|
|
raise ValueError("No list defined in json file: %s" % json_file)
|
|
return json_obj
|
|
|
|
|
|
def main(argv=sys.argv):
|
|
opts = parse_opts(argv)
|
|
if opts.print_templates:
|
|
print(opts.templates)
|
|
return 0
|
|
|
|
if not opts.metadata:
|
|
if 'OS_CONFIG_FILES' in os.environ:
|
|
opts.metadata = os.environ['OS_CONFIG_FILES'].split(':')
|
|
else:
|
|
opts.metadata = load_list_from_json(opts.os_config_files)
|
|
if ((not opts.metadata and opts.os_config_files ==
|
|
OS_CONFIG_FILES_PATH)):
|
|
logger.warning('DEPRECATED: falling back to %s' %
|
|
OS_CONFIG_FILES_PATH_OLD)
|
|
opts.metadata = load_list_from_json(OS_CONFIG_FILES_PATH_OLD)
|
|
|
|
if opts.key and opts.boolean_key:
|
|
logger.warning('--key is not compatible with --boolean-key.'
|
|
' --boolean-key ignored.')
|
|
|
|
try:
|
|
if opts.templates is None:
|
|
raise exc.ConfigException('missing option --templates')
|
|
|
|
if opts.key:
|
|
print_key(opts.metadata,
|
|
opts.key,
|
|
opts.type,
|
|
opts.key_default,
|
|
opts.fallback_metadata)
|
|
elif opts.boolean_key:
|
|
return boolean_key(opts.metadata,
|
|
opts.boolean_key,
|
|
opts.fallback_metadata)
|
|
else:
|
|
install_config(opts.metadata, opts.templates, opts.output,
|
|
opts.validate, opts.subhash, opts.fallback_metadata)
|
|
logger.info("success")
|
|
except exc.ConfigException as e:
|
|
logger.error(e)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
# logging
|
|
LOG_FORMAT = '[%(asctime)s] [%(levelname)s] %(message)s'
|
|
DATE_FORMAT = '%Y/%m/%d %I:%M:%S %p'
|
|
|
|
|
|
def add_handler(logger, handler):
|
|
handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT))
|
|
logger.addHandler(handler)
|
|
|
|
|
|
logger = logging.getLogger('os-apply-config')
|
|
logger.setLevel(logging.INFO)
|
|
add_handler(logger, logging.StreamHandler())
|
|
if os.geteuid() == 0:
|
|
add_handler(logger, logging.FileHandler('/var/log/os-apply-config.log'))
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main(sys.argv))
|