system-config/tools/remap_manifest.py
Monty Taylor b381b6e9bd Dump hiera values properly in remap
These changes were made live before the file split.

Change-Id: If7ac51fa2ffda5190ff2e04a3dc18161c3d30633
2015-10-28 12:31:11 +09:00

171 lines
5.8 KiB
Python
Executable File

#!/usr/bin/env python
# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This is a utility intended to split a single common.yaml file from
# a hieradata dir into a common.yaml / group/$::group.yaml / fqdn/$::fqdn.yaml
# structure.
# Values associated with the default node go into common.yaml
# Values associated with a node that has a group and is specified with a
# regex go into the group
# Values associated with a node that has a group but specifies a specific host
# will go into the fqdn file if they do not also appead in a group-related
# host.
#
# This utility is not intended to solve all problems and is not intended to be
# run on the same data twice. It's a one-off helper script, and it's
# potentially destructure. So be prepared to have a backup of your common.yaml
# file you can revert to after you inspect the output if it got something wrong
import collections
import pprint
import yaml
import os
groups = {}
fqdns = {}
common = []
current = None
current_keys = None
# from:
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data flake8: noqa
def should_use_block(value):
for c in u"\u000a\u000d\u001c\u001d\u001e\u0085\u2028\u2029":
if c in value:
return True
return False
def my_represent_scalar(self, tag, value, style=None):
if style is None:
if should_use_block(value):
style='|'
else:
style = self.default_style
node = yaml.representer.ScalarNode(tag, value, style=style)
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
return node
yaml.representer.BaseRepresenter.represent_scalar = my_represent_scalar
# end from
# from: http://pyyaml.org/ticket/64
class MyDumper(yaml.Dumper):
def increase_indent(self, flow=False, indentless=False):
return super(MyDumper, self).increase_indent(flow, False)
#end from
manifest_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'manifests/site.pp'))
with open(manifest_path, 'r') as manifest:
for line in manifest:
if line.startswith('#'):
continue
if 'node default' in line:
current_keys = common
continue
elif line.startswith('node'):
current_keys = list()
current = dict(keys=current_keys)
if '/' in line:
name = line.split('/')[1]
groups[name] = current
else:
name = line.split("'")[1]
fqdns[name] = current
continue
if '$group' in line:
if '"' in line:
name = line.split('"')[1]
elif "'" in line:
name = line.split("'")[1]
current['group'] = name
if 'hiera' in line:
key = line.split("'")[1]
if key not in common:
current_keys.append(key)
new_groups = {}
for value in groups.values():
new_groups[value['group']] = dict(keys=value['keys'])
groups = new_groups
# Trim group duplicates to just be in the group
for key, value in fqdns.items():
if 'group' in value:
new_keys = []
for possible_key in value['keys']:
if possible_key not in groups[value['group']]['keys']:
new_keys.append(possible_key)
value['keys'] = new_keys
# Print the values so that the person running can verify what's going on
pprint.pprint(dict(common=common, groups=groups, fqdns=fqdns))
def write_values(reverse_map, target, input_dict, source_vaues, root):
outdir = os.path.join(root, target)
if not os.path.exists(outdir):
os.makedirs(outdir)
for key, value in input_dict.items():
output_dict = {}
for name in value['keys']:
reverse_map[name].append(dict(target=target, key=key))
if name in source_values:
output_dict[name] = source_values[name]
else:
print "Requested key not in common.yaml", name
with open(os.path.join(outdir, '%s.yaml' % key), 'w') as outfile:
outfile.write(yaml.dump(
output_dict, default_flow_style=False, Dumper=MyDumper))
return reverse_map
def write_common_values(input_dict, source_values, root):
outdir = root
if not os.path.exists(outdir):
os.makedirs(outdir)
output_dict = {}
for name in input_dict:
output_dict[name] = source_values[name]
with open(os.path.join(outdir, 'common.yaml'), 'w') as outfile:
outfile.write(yaml.dump(output_dict, default_flow_style=False))
if os.path.exists('/etc/puppet/hieradata/production/common.yaml'):
source_values = yaml.load(
open('/etc/puppet/hieradata/production/common.yaml'))
root = '/etc/puppet/hieradata/production'
else:
def get_default():
return "Default data"
source_values = collections.defaultdict(get_default)
root = 'testoutput'
write_common_values(common, source_values, root)
reverse_map = collections.defaultdict(list)
reverse_map = write_values(reverse_map, 'fqdn', fqdns, source_values, root)
reverse_map = write_values(reverse_map, 'group', groups, source_values, root)
for key, value in reverse_map.items():
if len(value) > 1:
print "Key %s duplicated in %r" % (key, value)