a77d5ebde2
Set +x permission on files: * tools/convert_heat_nic_config_to_ansible_j2.py * tools/convert_v1_net_data.py Other python scripts have the executable permission set. Align the two scripts with the rest. Change-Id: I0a0bd4a353bdc35698444b72f093cce176bbe5a0
514 lines
20 KiB
Python
Executable File
514 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import argparse
|
|
import collections
|
|
import openstack
|
|
import os
|
|
import re
|
|
import sys
|
|
import yaml
|
|
|
|
|
|
MIN_VIABLE_MTU_HEADER = (
|
|
"{% set mtu_list = [ctlplane_mtu] %}\n"
|
|
"{% for network in role_networks %}\n"
|
|
"{{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}\n"
|
|
"{%- endfor %}\n"
|
|
"{% set min_viable_mtu = mtu_list | max %}\n"
|
|
)
|
|
|
|
DUAL_MIN_VIABLE_MTU_HEADER = (
|
|
"{% set mtu_ctlplane_list = [ctlplane_mtu] %}\n"
|
|
"{% set mtu_dataplane_list = [] %}\n"
|
|
"{% for network in role_networks %}\n"
|
|
"{# This block resolves the minimum viable MTU for interfaces connected to #}\n" # noqa
|
|
"{# the dataplane network(s), which start by Tenant, and also bonds #}\n"
|
|
"{# and bridges that carry multiple VLANs. Each VLAN may have different MTU. #}\n" # noqa
|
|
"{# The bridge, bond or interface must have an MTU to allow the VLAN with the #}\n" # noqa
|
|
"{# largest MTU. #}\n"
|
|
"{% if network.startswith('Tenant') %}\n"
|
|
"{{ mtu_dataplane_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}\n" # noqa
|
|
"{# This block resolves the minimum viable MTU for interfaces connected to #}\n" # noqa
|
|
"{# the control plane network(s) (don't start by Tenant), and also bonds #}\n" # noqa
|
|
"{# and bridges that carry multiple VLANs. Each VLAN may have different MTU. #}\n" # noqa
|
|
"{# The bridge, bond or interface must have an MTU to allow the VLAN with the #}\n" # noqa
|
|
"{# largest MTU. #}\n"
|
|
"{% else %}\n"
|
|
"{{ mtu_ctlplane_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}\n" # noqa
|
|
"{%- endif %}\n"
|
|
"{%- endfor %}\n"
|
|
"{% set min_viable_mtu_ctlplane = mtu_ctlplane_list | max %}\n"
|
|
"{% set min_viable_mtu_dataplane = mtu_dataplane_list | max %}\n"
|
|
)
|
|
|
|
UNSUPPORTED_HEAT_INTRINSIC_FUNCTIONS = {
|
|
'get_file', 'get_resource', 'digest', 'repeat', 'resource_facade',
|
|
'str_replace', 'str_replace_strict', 'str_split', 'map_merge',
|
|
'map_replace', 'yaql', 'equals', 'if', 'not', 'and', 'or', 'filter',
|
|
'make_url', 'contains'
|
|
}
|
|
|
|
QUOTE_FIX = '%_fix_quote_%'
|
|
DBL_QUOTE_FIX = '%_double_fix_quote_%'
|
|
|
|
|
|
def parse_opts(argv):
|
|
parser = argparse.ArgumentParser(
|
|
description='Convert to Ansible Jinja2 NIC config templates.')
|
|
parser.add_argument('--stack',
|
|
metavar='STACK_NAME',
|
|
help='Name or ID of heat stack (default=overcloud)',
|
|
default='overcloud')
|
|
parser.add_argument('-n', '--networks_file',
|
|
metavar='<network_data.yaml>',
|
|
required=True,
|
|
help=('Configuration file describing the network '
|
|
'deployment.'))
|
|
parser.add_argument('-y', '--yes',
|
|
default=False,
|
|
action='store_true',
|
|
help='Overwrite existing files.')
|
|
parser.add_argument('template',
|
|
metavar='TEMPLATE_FILE',
|
|
help='Existing NIC config template to convert.')
|
|
|
|
opts = parser.parse_args(argv[1:])
|
|
|
|
return opts
|
|
|
|
|
|
class TemplateDumper(yaml.SafeDumper):
|
|
def represent_ordered_dict(self, data):
|
|
return self.represent_dict(data.items())
|
|
|
|
|
|
class TemplateLoader(yaml.SafeLoader):
|
|
def construct_mapping(self, node):
|
|
self.flatten_mapping(node)
|
|
return collections.OrderedDict(self.construct_pairs(node))
|
|
|
|
|
|
TemplateDumper.add_representer(collections.OrderedDict,
|
|
TemplateDumper.represent_ordered_dict)
|
|
TemplateLoader.add_constructor(yaml.BaseLoader,
|
|
TemplateLoader.construct_mapping)
|
|
|
|
|
|
class ConvertToAnsibleJ2(object):
|
|
|
|
def __init__(self, stack_env, networks_file):
|
|
self.stack_env = stack_env
|
|
self.param_to_var_map = self.create_param_to_var_map(networks_file)
|
|
self.hard_coded_parameters = list()
|
|
|
|
@staticmethod
|
|
def unwrap_j2_var(x):
|
|
"""Strip jinja2 brackets from string
|
|
|
|
When nesting ansible vars in jinja2 the brackets must be
|
|
removed. This also adds appropriate quote fix prefix and
|
|
suffix.
|
|
"""
|
|
is_ansible_var = False
|
|
if isinstance(x, str):
|
|
if x.startswith('{{ ') and x.endswith(' }}'):
|
|
is_ansible_var = True
|
|
x = x[3:]
|
|
x = x[:-3]
|
|
else:
|
|
raise RuntimeError(
|
|
'Unsupported type {} for method unwrap_j2_var'.format(type(x)))
|
|
|
|
if is_ansible_var:
|
|
return DBL_QUOTE_FIX + '{}'.format(x) + DBL_QUOTE_FIX
|
|
else:
|
|
return QUOTE_FIX + '{}'.format(x) + QUOTE_FIX
|
|
|
|
@staticmethod
|
|
def strip_j2_comment(x):
|
|
"""Strip jinja2 comment from string
|
|
|
|
When nesting hard-coded parameter conversions in jinja2 the
|
|
comment must be removed.
|
|
"""
|
|
return re.sub('{#.*#}', '', x)
|
|
|
|
def normalize_complex(self, old):
|
|
if isinstance(old, list):
|
|
new = list()
|
|
for i in old:
|
|
if isinstance(i, str):
|
|
new.append(self.strip_j2_comment(self.unwrap_j2_var(i)))
|
|
if isinstance(i, (bool, int)):
|
|
new.append(i)
|
|
if isinstance(i, (list, dict)):
|
|
new.append(self.normalize_complex(i))
|
|
elif isinstance(old, dict):
|
|
new = dict()
|
|
for k, v in old.items():
|
|
k = QUOTE_FIX + '{}'.format(k) + QUOTE_FIX
|
|
if isinstance(v, str):
|
|
new[k] = self.strip_j2_comment(self.unwrap_j2_var(v))
|
|
if isinstance(v, (bool, int)):
|
|
new[k] = v
|
|
if isinstance(v, (list, dict)):
|
|
new[k] = self.normalize_complex(v)
|
|
else:
|
|
raise RuntimeError(
|
|
'Unsupported type {} for method normalize_complex'.format(
|
|
type(old)))
|
|
|
|
return new
|
|
|
|
@staticmethod
|
|
def to_j2_var(x):
|
|
if not isinstance(x, str):
|
|
raise RuntimeError(
|
|
'Unsupported type {} for method to_j2_var'.format(type(x)))
|
|
|
|
return '{{{{ {} }}}}'.format(x)
|
|
|
|
@staticmethod
|
|
def quote_fix(x):
|
|
return QUOTE_FIX + '{}'.format(x) + QUOTE_FIX
|
|
|
|
def create_param_to_var_map(self, networks_file):
|
|
_map = {
|
|
'ControlPlaneIp': self.to_j2_var('ctlplane_ip'),
|
|
'ControlPlaneSubnetCidr': self.to_j2_var('ctlplane_subnet_cidr'),
|
|
'ControlPlaneMtu': self.to_j2_var('ctlplane_mtu'),
|
|
'ControlPlaneDefaultRoute': self.to_j2_var('ctlplane_gateway_ip'),
|
|
'ControlPlaneStaticRoutes': self.to_j2_var('ctlplane_host_routes'),
|
|
'DnsServers': self.to_j2_var('ctlplane_dns_nameservers'),
|
|
'DnsSearchDomains': self.to_j2_var('dns_search_domains'),
|
|
'NumDpdkInterfaceRxQueues':
|
|
self.to_j2_var('num_dpdk_interface_rx_queues'),
|
|
'BondInterfaceOvsOptions':
|
|
self.to_j2_var('bond_interface_ovs_options')
|
|
}
|
|
|
|
with open(networks_file, 'r') as f:
|
|
networks = yaml.safe_load(f.read())
|
|
|
|
for net in networks:
|
|
name = net['name']
|
|
name_lower = net.get('name_lower', net['name'].lower())
|
|
_map.update({
|
|
name + 'NetworkVlanID':
|
|
self.to_j2_var('{}_vlan_id'.format(name_lower)),
|
|
name + 'IpSubnet':
|
|
'/'.join([self.to_j2_var('{}_ip'.format(name_lower)),
|
|
self.to_j2_var('{}_cidr'.format(name_lower))]),
|
|
name + 'InterfaceDefaultRoute':
|
|
self.to_j2_var('{}_gateway_ip'.format(name_lower)),
|
|
name + 'InterfaceRoutes':
|
|
self.to_j2_var('{}_host_routes'.format(name_lower)),
|
|
name + 'Mtu':
|
|
self.to_j2_var('{}_mtu'.format(name_lower))
|
|
})
|
|
|
|
return _map
|
|
|
|
def convert_get_param(self, old):
|
|
param = old['get_param']
|
|
if isinstance(param, str):
|
|
if param in self.param_to_var_map:
|
|
return self.param_to_var_map[param]
|
|
elif param in self.stack_env.get('parameter_defaults', {}):
|
|
stack_value = self.stack_env['parameter_defaults'][param]
|
|
print('INFO - Custom Parameter {} was hard-coded in the '
|
|
'converted template using the value from the Heat stack '
|
|
'environment.\n'
|
|
' To parameterize the value an ansible var must be '
|
|
'added using the {{role.name}}ExtraGroupVars '
|
|
'THT interface and the template modified to use the '
|
|
'ansible var.'.format(param))
|
|
j2_comment = (
|
|
'{{# NOTE! Custom parameter {} was hard-coded in '
|
|
'the converted template. To parameterize use the '
|
|
'{{role.name}}ExtraGroupVars THT interface and update the '
|
|
'template to use an ansible var. #}}'.format(param))
|
|
self.hard_coded_parameters.append(param)
|
|
if isinstance(stack_value, str):
|
|
return self.quote_fix(stack_value + j2_comment)
|
|
else:
|
|
return stack_value
|
|
else:
|
|
print('WARNING - Manual intervention required. Unable to '
|
|
'convert get_param occurrence: {}'.format(old))
|
|
return self.quote_fix(
|
|
'NEED MANUAL CONVERSION: {}'.format(str(old)))
|
|
elif isinstance(param, list):
|
|
print('WARNING - can not convert get_param referencing values in '
|
|
'complex datastructures. Please review the Ansible Jinja2 '
|
|
'template and convert this manually.')
|
|
return self.quote_fix(
|
|
'NEED MANUAL CONVERSION: {}'.format(str(old)))
|
|
|
|
raise RuntimeError(
|
|
'Unexpected Type {} in get_param: {}'.format(type(param), old))
|
|
|
|
def convert_get_attr(self, old):
|
|
attr = old['get_attr']
|
|
if not isinstance(attr, list):
|
|
raise RuntimeError(
|
|
'Attributes for get_attr convertsion must of type list.')
|
|
|
|
if 'MinViableMtu' in attr:
|
|
return self.to_j2_var('min_viable_mtu')
|
|
elif 'MinViableMtuBondApi' in attr:
|
|
return self.to_j2_var('min_viable_mtu_ctlplane')
|
|
elif 'MinViableMtuBondData' in attr:
|
|
return self.to_j2_var('min_viable_mtu_dataplane')
|
|
else:
|
|
print('WARNING - only MinViableMtu and combined '
|
|
'MinViableMtuBondApi + MinViableMtuBondData attribute can '
|
|
'be converted. Please review the Ansible Jinja2 template '
|
|
'and convert this manually.')
|
|
return 'NEED MANUAL CONVERSION: {}'.format(str(old))
|
|
|
|
def convert_list_join(self, list_join_attrs):
|
|
to_join = list()
|
|
for x in list_join_attrs[1]:
|
|
to_join.append(self.recursive_convert(x))
|
|
|
|
return list_join_attrs[0].join(to_join)
|
|
|
|
def convert_list_concat(self, list_concat_attrs):
|
|
ansible_concat_tpl = "{} | flatten"
|
|
to_concatenate = list()
|
|
if not isinstance(list_concat_attrs, list):
|
|
raise RuntimeError('list_concat_args must be a list')
|
|
|
|
for x in list_concat_attrs:
|
|
to_concatenate.append(self.recursive_convert(x))
|
|
|
|
to_concatenate = self.normalize_complex(to_concatenate)
|
|
new = ansible_concat_tpl.format(to_concatenate)
|
|
|
|
return self.to_j2_var(new)
|
|
|
|
def convert_list_concat_unique(self, list_concat_unique_attrs):
|
|
ansible_concat_unique_tpl = "{} | flatten | unique"
|
|
to_concatenate = list()
|
|
if not isinstance(list_concat_unique_attrs, list):
|
|
raise RuntimeError('list_concat_unique_attrs must be a list')
|
|
|
|
for x in list_concat_unique_attrs:
|
|
to_concatenate.append(self.recursive_convert(x))
|
|
|
|
to_concatenate = self.normalize_complex(to_concatenate)
|
|
new = ansible_concat_unique_tpl.format(to_concatenate)
|
|
|
|
return self.to_j2_var(new)
|
|
|
|
@staticmethod
|
|
def convert_unsupported(old, fn):
|
|
print('WARNING - can not convert unsupported heat intrinsic function '
|
|
' {}. Please review the Ansible Jinja2 template and convert '
|
|
'this manually.'.format(fn))
|
|
return ('UNSUPPORTED HEAT INTRINSIC FUNCTION {} '
|
|
'REQUIRES MANUAL CONVERSION {}'.format(fn, old))
|
|
|
|
def recursive_convert(self, old):
|
|
if isinstance(old, (bool, str, int)):
|
|
new = old
|
|
elif isinstance(old, dict):
|
|
for fn in UNSUPPORTED_HEAT_INTRINSIC_FUNCTIONS:
|
|
if fn in old:
|
|
return self.convert_unsupported(old, fn)
|
|
|
|
if 'get_param' in old:
|
|
return self.convert_get_param(old)
|
|
elif 'get_attr' in old:
|
|
return self.convert_get_attr(old)
|
|
elif 'list_join' in old:
|
|
return self.convert_list_join(old['list_join'])
|
|
elif 'list_concat_unique' in old:
|
|
return self.convert_list_concat_unique(
|
|
old['list_concat_unique'])
|
|
else:
|
|
new = collections.OrderedDict()
|
|
for k, v in old.items():
|
|
if isinstance(v, (bool, str, int)):
|
|
new[k] = v
|
|
elif isinstance(v, (list, dict)):
|
|
new[k] = self.recursive_convert(v)
|
|
else:
|
|
raise RuntimeError(
|
|
'Unexpected Type {} for key: {}'.format(
|
|
type(v), k))
|
|
|
|
elif isinstance(old, list):
|
|
new = list()
|
|
for x in old:
|
|
new.append(self.recursive_convert(x))
|
|
else:
|
|
raise RuntimeError(
|
|
'Unknown type {} for recursive convert'.format(type(old)))
|
|
|
|
return new
|
|
|
|
def convert_template(self, template):
|
|
with open(template, 'r') as f:
|
|
heat_tpl = yaml.safe_load(f.read())
|
|
|
|
resources = set(heat_tpl['resources'].keys())
|
|
|
|
if not resources.issubset({'OsNetConfigImpl',
|
|
'MinViableMtu',
|
|
'MinViableMtuBondApi',
|
|
'MinViableMtuBondData'}):
|
|
msg = ('Only OsNetConfigImpl and MinViableMtu resoureces '
|
|
'supported. Found resources: {}'.format(resources))
|
|
raise RuntimeError(msg)
|
|
|
|
net_config_res = heat_tpl['resources'].get('OsNetConfigImpl')
|
|
mtu_header = None
|
|
if heat_tpl['resources'].get('MinViableMtu'):
|
|
mtu_header = 'single'
|
|
elif (heat_tpl['resources'].get('MinViableMtuBondApi')
|
|
and heat_tpl['resources'].get('MinViableMtuBondData')):
|
|
mtu_header = 'dual'
|
|
|
|
if not net_config_res:
|
|
raise RuntimeError('OsNetConfigImpl resource not found in '
|
|
'template.')
|
|
|
|
net_config_res_props = net_config_res['properties']
|
|
|
|
if net_config_res['type'] == 'OS::Heat::Value':
|
|
h_net_conf = net_config_res_props['value']
|
|
elif net_config_res['type'] == 'OS::Heat::SoftwareConfig':
|
|
h_net_conf = net_config_res_props['config']['str_replace'][
|
|
'params']['$network_config']['network_config']
|
|
else:
|
|
raise RuntimeError('No network config found in provided template.')
|
|
|
|
j2_config = collections.OrderedDict({'network_config': []})
|
|
j2_net_conf = j2_config['network_config']
|
|
|
|
j2_net_conf.extend(self.recursive_convert(h_net_conf))
|
|
|
|
j2_header = None
|
|
if self.hard_coded_parameters:
|
|
j2_header = (
|
|
'{# The values of the following custom heat parameters was '
|
|
'hard-coded into this template:\n')
|
|
for param in self.hard_coded_parameters:
|
|
j2_header += ' * {}\n'.format(param)
|
|
j2_header += (
|
|
'To parameterize use the {{role.name}}ExtraGroupVars THT '
|
|
'interface and update the template to use an ansible var.\n'
|
|
'#}\n')
|
|
|
|
return j2_config, mtu_header, j2_header
|
|
|
|
|
|
def write_j2_template(j2_template, j2_config, mtu_header, j2_header):
|
|
"""Write the Jinja2 template file
|
|
|
|
This is done in three steps because the YAML dumper insists
|
|
to add quotes:
|
|
write the template file
|
|
read the template file
|
|
re-write it with quotes removed
|
|
"""
|
|
with open(j2_template, 'w') as f:
|
|
if j2_header:
|
|
f.write(j2_header)
|
|
f.write('---\n')
|
|
if mtu_header:
|
|
if mtu_header == 'single':
|
|
f.write(MIN_VIABLE_MTU_HEADER)
|
|
elif mtu_header == 'dual':
|
|
f.write(DUAL_MIN_VIABLE_MTU_HEADER)
|
|
yaml.dump(j2_config, f, TemplateDumper, width=256,
|
|
default_flow_style=False)
|
|
|
|
with open(j2_template, 'r') as f:
|
|
data = f.read()
|
|
# Remove quote before jinja2 var reference
|
|
data = data.replace('\'{{', '{{')
|
|
# Remove quote after jinja2 var reference
|
|
data = data.replace('}}\'', '}}')
|
|
# Remove quote before value imported from stack environment
|
|
data = data.replace('\'' + QUOTE_FIX, '')
|
|
# Remove quote after value imported from stack environment
|
|
data = data.replace(QUOTE_FIX + '\'', '')
|
|
# Remove quote before value imported from stack environment
|
|
data = data.replace('\'\'' + DBL_QUOTE_FIX, '')
|
|
# Remove quote after value imported from stack environment
|
|
data = data.replace(DBL_QUOTE_FIX + '\'\'', '')
|
|
# Remove quote_fix string
|
|
data = data.replace(QUOTE_FIX, '')
|
|
data = data.replace(DBL_QUOTE_FIX, '')
|
|
|
|
with open(j2_template, 'w') as f:
|
|
f.write(data)
|
|
|
|
|
|
def validate_files(opts, template, networks_file, j2_template):
|
|
if not os.path.exists(template):
|
|
raise RuntimeError('Template file not found {}.'.format(template))
|
|
if not os.path.isfile(template):
|
|
raise RuntimeError('Template {} is not a file.'.format(template))
|
|
if not os.path.exists(networks_file):
|
|
raise RuntimeError('Networks file not found, {}.'.format(
|
|
networks_file))
|
|
if not os.path.isfile(networks_file):
|
|
raise RuntimeError('Networks file {} is not a file.'.format(
|
|
networks_file))
|
|
if os.path.exists(j2_template) and not opts.yes:
|
|
raise RuntimeError('Ansible Jinja2 template {} already exists'.format(
|
|
j2_template))
|
|
if os.path.exists(j2_template) and not os.path.isfile(j2_template):
|
|
raise RuntimeError('Existing {} is not a file.'.format(j2_template))
|
|
pass
|
|
|
|
|
|
def get_stack_environment(stack_name):
|
|
try:
|
|
conn = openstack.connect('undercloud')
|
|
stack = conn.orchestration.find_stack(stack_name)
|
|
if not stack:
|
|
print('INFO: Heat stack {} not found.'.format(stack_name))
|
|
return {}
|
|
stack_env = conn.orchestration.get_stack_environment(stack)
|
|
except Exception as e:
|
|
print('ERROR: Unable to get stack environment.')
|
|
raise e
|
|
|
|
return stack_env
|
|
|
|
|
|
def main():
|
|
opts = parse_opts(sys.argv)
|
|
|
|
template = os.path.abspath(opts.template)
|
|
networks_file = os.path.abspath(opts.networks_file)
|
|
j2_template = os.path.splitext(template)[0] + '.j2'
|
|
validate_files(opts, template, networks_file, j2_template)
|
|
|
|
stack_env = get_stack_environment(opts.stack)
|
|
|
|
converter = ConvertToAnsibleJ2(stack_env, networks_file)
|
|
|
|
j2_config, mtu_header, j2_header = converter.convert_template(template)
|
|
write_j2_template(j2_template, j2_config, mtu_header, j2_header)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|