3815c906d4
Although this only needs to be done once, it's ugly enough of a task that it should have a script, and that script should be reviewed. The script can be run locally on a machine without a hieradata and it will create a local directory with all of the keys and none of the data. It will also print out the total mapping and a report on which keys are duplicated across hosts after group de-duplication is taken in to account. Change-Id: Id0af46abb8f4167bd91e95cd7a6cd2e35e5b0a94
137 lines
4.6 KiB
Python
Executable File
137 lines
4.6 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# This is a utility intended to split a single common.yaml file from
|
|
# a hieradata dir into a common.yaml / group/$::group.yaml / fqdn/$::fqdn.yaml
|
|
# structure.
|
|
# Values associated with the default node go into common.yaml
|
|
# Values associated with a node that has a group and is specified with a
|
|
# regex go into the group
|
|
# Values associated with a node that has a group but specifies a specific host
|
|
# will go into the fqdn file if they do not also appead in a group-related
|
|
# host.
|
|
#
|
|
# This utility is not intended to solve all problems and is not intended to be
|
|
# run on the same data twice. It's a one-off helper script, and it's
|
|
# potentially destructure. So be prepared to have a backup of your common.yaml
|
|
# file you can revert to after you inspect the output if it got something wrong
|
|
|
|
import collections
|
|
import pprint
|
|
import yaml
|
|
import os
|
|
|
|
|
|
groups = {}
|
|
fqdns = {}
|
|
common = []
|
|
|
|
current = None
|
|
current_keys = None
|
|
|
|
manifest_path = os.path.abspath(
|
|
os.path.join(os.path.dirname(__file__), '..', 'manifests/site.pp'))
|
|
|
|
with open(manifest_path, 'r') as manifest:
|
|
for line in manifest:
|
|
if line.startswith('#'):
|
|
continue
|
|
if 'node default' in line:
|
|
current_keys = common
|
|
continue
|
|
elif line.startswith('node'):
|
|
current_keys = list()
|
|
current = dict(keys=current_keys)
|
|
if '/' in line:
|
|
name = line.split('/')[1]
|
|
groups[name] = current
|
|
else:
|
|
name = line.split("'")[1]
|
|
fqdns[name] = current
|
|
continue
|
|
if '$group' in line:
|
|
name = line.split('"')[1]
|
|
current['group'] = name
|
|
if 'hiera' in line:
|
|
key = line.split("'")[1]
|
|
if key not in common:
|
|
current_keys.append(key)
|
|
|
|
|
|
new_groups = {}
|
|
for value in groups.values():
|
|
new_groups[value['group']] = dict(keys=value['keys'])
|
|
groups = new_groups
|
|
|
|
# Trim group duplicates to just be in the group
|
|
for key, value in fqdns.items():
|
|
if 'group' in value:
|
|
new_keys = []
|
|
for possible_key in value['keys']:
|
|
if possible_key not in groups[value['group']]['keys']:
|
|
new_keys.append(possible_key)
|
|
value['keys'] = new_keys
|
|
|
|
# Print the values so that the person running can verify what's going on
|
|
pprint.pprint(dict(common=common, groups=groups, fqdns=fqdns))
|
|
|
|
|
|
def write_values(reverse_map, target, input_dict, source_vaues, root):
|
|
outdir = os.path.join(root, target)
|
|
if not os.path.exists(outdir):
|
|
os.makedirs(outdir)
|
|
for key, value in input_dict.items():
|
|
output_dict = {}
|
|
for name in value['keys']:
|
|
reverse_map[name].append(dict(target=target, key=key))
|
|
output_dict[name] = source_values[name]
|
|
with open(os.path.join(outdir, '%s.yaml' % key), 'w') as outfile:
|
|
outfile.write(yaml.dump(output_dict, default_flow_style=False))
|
|
return reverse_map
|
|
|
|
|
|
def write_common_values(input_dict, source_values, root):
|
|
outdir = root
|
|
if not os.path.exists(outdir):
|
|
os.makedirs(outdir)
|
|
output_dict = {}
|
|
for name in input_dict:
|
|
output_dict[name] = source_values[name]
|
|
with open(os.path.join(outdir, 'common.yaml'), 'w') as outfile:
|
|
outfile.write(yaml.dump(output_dict, default_flow_style=False))
|
|
|
|
|
|
if os.path.exists('/etc/puppet/hieradata/production/common.yaml'):
|
|
source_values = yaml.load(
|
|
open('/etc/puppet/hieradata/production/common.yaml'))
|
|
root = '/etc/puppet/hieradata/production'
|
|
else:
|
|
def get_default():
|
|
return "Default data"
|
|
source_values = collections.defaultdict(get_default)
|
|
root = 'testoutput'
|
|
|
|
|
|
write_common_values(common, source_values, root)
|
|
reverse_map = collections.defaultdict(list)
|
|
reverse_map = write_values(reverse_map, 'fqdn', fqdns, source_values, root)
|
|
reverse_map = write_values(reverse_map, 'group', groups, source_values, root)
|
|
|
|
|
|
for key, value in reverse_map.items():
|
|
if len(value) > 1:
|
|
print "Key %s duplicated in %r" % (key, value)
|