Merge "Allow rules to be input by user"
This commit is contained in:
Zuul
Gerrit Code Review
commit
c1be6eb7e3
@ -56,6 +56,11 @@ engineering Excel files. Must be a readable file in YAML format.
|
||||
|
||||
Path to site specific configuration YAML. Must be a readable file.
|
||||
|
||||
**-r / \\-\\-rule-configuration** (Optional).
|
||||
|
||||
Path to rules configuration YAML file. This file defines the rules used for
|
||||
data manipulation. Default rules are used if no rules YAML is entered.
|
||||
|
||||
**\\-\\-intermediary-schema** (Optional).
|
||||
|
||||
Path to the intermediary schema to be used for validation.
|
||||
@ -107,6 +112,11 @@ engineering Excel files. Must be a readable file in YAML format.
|
||||
|
||||
Path to site specific configuration YAML. Must be a readable file.
|
||||
|
||||
**-r / \\-\\-rule-configuration** (Optional).
|
||||
|
||||
Path to rules configuration YAML file. This file defines the rules used for
|
||||
data manipulation. Default rules are used if no rules YAML is entered.
|
||||
|
||||
**\\-\\-intermediary-schema** (Optional).
|
||||
|
||||
Path to the intermediary schema to be used for validation.
|
||||
|
||||
@ -42,6 +42,14 @@ SITE_CONFIGURATION_FILE_OPTION = click.option(
|
||||
required=False,
|
||||
help='Path to site specific configuration details YAML file.')
|
||||
|
||||
RULE_CONFIGURATION_FILE_OPTION = click.option(
|
||||
'-r',
|
||||
'--rule-configuration',
|
||||
'rule_configuration',
|
||||
type=click.Path(exists=True, readable=True, dir_okay=False),
|
||||
required=False,
|
||||
help='Path to data manipulation configuration rules YAML file.')
|
||||
|
||||
INTERMEDIARY_DIR_OPTION = click.option(
|
||||
'-d',
|
||||
'--intermediary-dir',
|
||||
@ -151,6 +159,7 @@ def intermediary_processor(plugin_type, **kwargs):
|
||||
LOG.info("Apply design rules to the extracted data")
|
||||
process_input_ob = ProcessDataSource(
|
||||
kwargs['site_name'], data_extractor.data,
|
||||
kwargs.get('rule_configuration', None),
|
||||
kwargs.get('intermediary_schema', None),
|
||||
kwargs.get('no_validation', False))
|
||||
return process_input_ob
|
||||
|
||||
38
spyglass/examples/rules.yaml
Normal file
38
spyglass/examples/rules.yaml
Normal file
@ -0,0 +1,38 @@
|
||||
###########################
|
||||
# Global Rules #
|
||||
###########################
|
||||
#Rule1: ip_alloc_offset
|
||||
# Specifies the number of ip addresses to offset from
|
||||
# the start of subnet allocation pool while allocating it to host.
|
||||
# -for vlan it is set to 12 as default.
|
||||
# -for oob it is 10
|
||||
# -for all gateway ip addresss it is set to 1.
|
||||
# -for ingress vip it is 1
|
||||
# -for static end (non pxe) it is -1( means one but last ip of the pool)
|
||||
# -for dhcp end (pxe only) it is -2( 3rd from the last ip of the pool)
|
||||
#Rule2: host_profile_interfaces.
|
||||
# Specifies the network interfaces type and
|
||||
# and their names for a particular hw profile
|
||||
#Rule3: hardware_profile
|
||||
# This specifies the profile details bases on sitetype.
|
||||
# It specifies the profile name and host type for compute,
|
||||
# controller along with hw type
|
||||
---
|
||||
rule_ip_alloc_offset:
|
||||
name: ip_alloc_offset
|
||||
ip_alloc_offset:
|
||||
default: 12
|
||||
oob: 10
|
||||
gateway: 1
|
||||
ingress_vip: 1
|
||||
static_ip_end: -2
|
||||
dhcp_ip_end: -2
|
||||
rule_hardware_profile:
|
||||
name: hardware_profile
|
||||
hardware_profile:
|
||||
foundry:
|
||||
profile_name:
|
||||
compute: dp-r720
|
||||
ctrl: cp-r720
|
||||
hw_type: dell_r720
|
||||
...
|
||||
@ -32,6 +32,7 @@ class ProcessDataSource(object):
|
||||
self,
|
||||
region,
|
||||
extracted_data,
|
||||
rules_config,
|
||||
intermediary_schema=None,
|
||||
no_validation=True):
|
||||
# Initialize intermediary and save site type
|
||||
@ -40,6 +41,7 @@ class ProcessDataSource(object):
|
||||
self.genesis_node = None
|
||||
self.network_subnets = None
|
||||
self.region_name = region
|
||||
self.rules = rules_config
|
||||
self.no_validation = no_validation
|
||||
if intermediary_schema and not self.no_validation:
|
||||
with open(intermediary_schema, 'r') as loaded_schema:
|
||||
@ -110,14 +112,16 @@ class ProcessDataSource(object):
|
||||
information. The method calls corresponding rule handler function
|
||||
based on rule name and applies them to appropriate data objects.
|
||||
"""
|
||||
|
||||
LOG.info("Apply design rules")
|
||||
# TODO(ian-pittwood): We may want to let users specify these in cli
|
||||
# opts. We also need better guidelines over how
|
||||
# to write these rules and how they are applied.
|
||||
|
||||
if self.rules is None:
|
||||
LOG.info("Apply design rules: Default")
|
||||
rules_dir = resource_filename("spyglass", "config/")
|
||||
rules_file = os.path.join(rules_dir, "rules.yaml")
|
||||
else:
|
||||
LOG.info("Apply design rules: " + str(self.rules))
|
||||
rules_file = self.rules
|
||||
rules_data_raw = self._read_file(rules_file)
|
||||
rules_yaml = yaml.safe_load(rules_data_raw)
|
||||
for rule in rules_yaml.keys():
|
||||
|
||||
@ -32,10 +32,24 @@ FIXTURE_DIR = os.path.join(
|
||||
@mark.usefixtures('rules_data')
|
||||
class TestProcessDataSource(unittest.TestCase):
|
||||
REGION_NAME = 'test'
|
||||
DEFAULT_RULES = None
|
||||
INPUT_RULES = os.path.join(FIXTURE_DIR, 'rules.yaml')
|
||||
|
||||
def test___init__(self):
|
||||
expected_data = 'data'
|
||||
obj = ProcessDataSource(self.REGION_NAME, expected_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, expected_data, self.DEFAULT_RULES)
|
||||
self.assertEqual(self.REGION_NAME, obj.region_name)
|
||||
self.assertDictEqual({}, obj.host_type)
|
||||
self.assertEqual(expected_data, obj.data)
|
||||
self.assertIsNone(obj.sitetype)
|
||||
self.assertIsNone(obj.genesis_node)
|
||||
self.assertIsNone(obj.network_subnets)
|
||||
|
||||
def test___init__rules_input(self):
|
||||
expected_data = 'data'
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, expected_data, self.INPUT_RULES)
|
||||
self.assertEqual(self.REGION_NAME, obj.region_name)
|
||||
self.assertDictEqual({}, obj.host_type)
|
||||
self.assertEqual(expected_data, obj.data)
|
||||
@ -59,7 +73,22 @@ class TestProcessDataSource(unittest.TestCase):
|
||||
'pxe': IPNetwork('30.30.4.0/25'),
|
||||
'storage': IPNetwork('30.31.1.0/25')
|
||||
}
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
result = obj._get_network_subnets()
|
||||
self.assertDictEqual(expected_result, result)
|
||||
|
||||
def test__get_network_subnets_input_rules(self):
|
||||
expected_result = {
|
||||
'calico': IPNetwork('30.29.1.0/25'),
|
||||
'oam': IPNetwork('10.0.220.0/26'),
|
||||
'oob': IPNetwork('10.0.220.128/27'),
|
||||
'overlay': IPNetwork('30.19.0.0/25'),
|
||||
'pxe': IPNetwork('30.30.4.0/25'),
|
||||
'storage': IPNetwork('30.31.1.0/25')
|
||||
}
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
result = obj._get_network_subnets()
|
||||
self.assertDictEqual(expected_result, result)
|
||||
|
||||
@ -67,22 +96,49 @@ class TestProcessDataSource(unittest.TestCase):
|
||||
expected_result = self.site_document_data.get_baremetal_host_by_type(
|
||||
'genesis')[0]
|
||||
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
obj._get_genesis_node_details()
|
||||
self.assertEqual(expected_result, obj.genesis_node)
|
||||
|
||||
def test__get_genesis_node_details_input_rules(self):
|
||||
expected_result = self.site_document_data.get_baremetal_host_by_type(
|
||||
'genesis')[0]
|
||||
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
obj._get_genesis_node_details()
|
||||
self.assertEqual(expected_result, obj.genesis_node)
|
||||
|
||||
def test__validate_intermediary_data(self):
|
||||
schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, schema_path, False)
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES,
|
||||
schema_path, False)
|
||||
result = obj._validate_intermediary_data()
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test__validate_intermediary_data_input_rules(self):
|
||||
schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES,
|
||||
schema_path, False)
|
||||
result = obj._validate_intermediary_data()
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test__validate_intermediary_data_invalid(self):
|
||||
schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.invalid_site_document_data, schema_path,
|
||||
False)
|
||||
self.REGION_NAME, self.invalid_site_document_data,
|
||||
self.DEFAULT_RULES, schema_path, False)
|
||||
with self.assertRaises(exceptions.IntermediaryValidationException):
|
||||
obj._validate_intermediary_data()
|
||||
|
||||
def test__validate_intermediary_data_invalid_input_rules(self):
|
||||
schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.invalid_site_document_data,
|
||||
self.INPUT_RULES, schema_path, False)
|
||||
with self.assertRaises(exceptions.IntermediaryValidationException):
|
||||
obj._validate_intermediary_data()
|
||||
|
||||
@ -90,7 +146,18 @@ class TestProcessDataSource(unittest.TestCase):
|
||||
@mock.patch.object(ProcessDataSource, '_apply_rule_hardware_profile')
|
||||
def test__apply_design_rules(
|
||||
self, mock_rule_hw_profile, mock_rule_ip_alloc_offset):
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
obj._apply_design_rules()
|
||||
mock_rule_hw_profile.assert_called_once()
|
||||
mock_rule_ip_alloc_offset.assert_called_once()
|
||||
|
||||
@mock.patch.object(ProcessDataSource, '_apply_rule_ip_alloc_offset')
|
||||
@mock.patch.object(ProcessDataSource, '_apply_rule_hardware_profile')
|
||||
def test__apply_design_rules_input_rules(
|
||||
self, mock_rule_hw_profile, mock_rule_ip_alloc_offset):
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
obj._apply_design_rules()
|
||||
mock_rule_hw_profile.assert_called_once()
|
||||
mock_rule_ip_alloc_offset.assert_called_once()
|
||||
@ -99,7 +166,28 @@ class TestProcessDataSource(unittest.TestCase):
|
||||
input_rules = self.rules_data['rule_hardware_profile'][
|
||||
'hardware_profile']
|
||||
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
obj._apply_rule_hardware_profile(input_rules)
|
||||
self.assertEqual(
|
||||
1, len(obj.data.get_baremetal_host_by_type('genesis')))
|
||||
self.assertEqual(
|
||||
3, len(obj.data.get_baremetal_host_by_type('controller')))
|
||||
self.assertEqual(
|
||||
8, len(obj.data.get_baremetal_host_by_type('compute')))
|
||||
for host in obj.data.get_baremetal_host_by_type('genesis'):
|
||||
self.assertEqual('cp-r720', host.host_profile)
|
||||
for host in obj.data.get_baremetal_host_by_type('controller'):
|
||||
self.assertEqual('cp-r720', host.host_profile)
|
||||
for host in obj.data.get_baremetal_host_by_type('compute'):
|
||||
self.assertEqual('dp-r720', host.host_profile)
|
||||
|
||||
def test__apply_rule_hardware_profile_input_rules(self):
|
||||
input_rules = self.rules_data['rule_hardware_profile'][
|
||||
'hardware_profile']
|
||||
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
obj._apply_rule_hardware_profile(input_rules)
|
||||
self.assertEqual(
|
||||
1, len(obj.data.get_baremetal_host_by_type('genesis')))
|
||||
@ -121,7 +209,24 @@ class TestProcessDataSource(unittest.TestCase):
|
||||
def test__apply_rule_ip_alloc_offset(
|
||||
self, mock__get_network_subnets, mock__update_vlan_net_data,
|
||||
mock__update_baremetal_host_ip_data):
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
obj._apply_rule_ip_alloc_offset(self.rules_data)
|
||||
self.assertEqual('success', obj.network_subnets)
|
||||
mock__get_network_subnets.assert_called_once()
|
||||
mock__update_vlan_net_data.assert_called_once_with(self.rules_data)
|
||||
mock__update_baremetal_host_ip_data.assert_called_once_with(
|
||||
self.rules_data)
|
||||
|
||||
@mock.patch.object(ProcessDataSource, '_update_baremetal_host_ip_data')
|
||||
@mock.patch.object(ProcessDataSource, '_update_vlan_net_data')
|
||||
@mock.patch.object(
|
||||
ProcessDataSource, '_get_network_subnets', return_value='success')
|
||||
def test__apply_rule_ip_alloc_offset_input_rules(
|
||||
self, mock__get_network_subnets, mock__update_vlan_net_data,
|
||||
mock__update_baremetal_host_ip_data):
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
obj._apply_rule_ip_alloc_offset(self.rules_data)
|
||||
self.assertEqual('success', obj.network_subnets)
|
||||
mock__get_network_subnets.assert_called_once()
|
||||
@ -130,7 +235,26 @@ class TestProcessDataSource(unittest.TestCase):
|
||||
self.rules_data)
|
||||
|
||||
def test__update_baremetal_host_ip_data(self):
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
obj.network_subnets = obj._get_network_subnets()
|
||||
ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][
|
||||
'ip_alloc_offset']
|
||||
obj._update_baremetal_host_ip_data(ip_alloc_offset_rules)
|
||||
|
||||
counter = 0
|
||||
for rack in obj.data.baremetal:
|
||||
for host in rack.hosts:
|
||||
for net_type, net_ip in iter(host.ip):
|
||||
ips = list(obj.network_subnets[net_type])
|
||||
self.assertEqual(
|
||||
str(ips[counter + ip_alloc_offset_rules['default']]),
|
||||
net_ip)
|
||||
counter += 1
|
||||
|
||||
def test__update_baremetal_host_ip_data_input_rules(self):
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
obj.network_subnets = obj._get_network_subnets()
|
||||
ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][
|
||||
'ip_alloc_offset']
|
||||
@ -150,7 +274,59 @@ class TestProcessDataSource(unittest.TestCase):
|
||||
ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][
|
||||
'ip_alloc_offset']
|
||||
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
obj.network_subnets = obj._get_network_subnets()
|
||||
obj._update_vlan_net_data(ip_alloc_offset_rules)
|
||||
|
||||
ingress_data = obj.data.network.get_vlan_data_by_name('ingress')
|
||||
subnet = IPNetwork(ingress_data.subnet[0])
|
||||
ips = list(subnet)
|
||||
self.assertEqual(
|
||||
str(ips[ip_alloc_offset_rules['ingress_vip']]),
|
||||
obj.data.network.bgp['ingress_vip'])
|
||||
self.assertEqual(
|
||||
ingress_data.subnet[0],
|
||||
obj.data.network.bgp['public_service_cidr'])
|
||||
subnets = obj.network_subnets
|
||||
for vlan in self.site_document_data.network.vlan_network_data:
|
||||
if vlan.role == 'ingress':
|
||||
continue
|
||||
ips = list(subnets[vlan.role])
|
||||
self.assertEqual(
|
||||
str(ips[ip_alloc_offset_rules['gateway']]), vlan.gateway)
|
||||
|
||||
if vlan.role == 'oob':
|
||||
ip_offset = ip_alloc_offset_rules['oob']
|
||||
else:
|
||||
ip_offset = ip_alloc_offset_rules['default']
|
||||
self.assertEqual(str(ips[1]), vlan.reserved_start)
|
||||
self.assertEqual(str(ips[ip_offset]), vlan.reserved_end)
|
||||
self.assertEqual(str(ips[ip_offset + 1]), vlan.static_start)
|
||||
|
||||
if vlan.role == 'pxe':
|
||||
self.assertEqual(
|
||||
str(ips[(len(ips) // 2) - 1]), vlan.static_end)
|
||||
self.assertEqual(str(ips[len(ips) // 2]), vlan.dhcp_start)
|
||||
self.assertEqual(
|
||||
str(ips[ip_alloc_offset_rules['dhcp_ip_end']]),
|
||||
vlan.dhcp_end)
|
||||
else:
|
||||
self.assertEqual(
|
||||
str(ips[ip_alloc_offset_rules['static_ip_end']]),
|
||||
vlan.static_end)
|
||||
|
||||
if vlan.role == 'oam':
|
||||
self.assertEqual(['0.0.0.0/0'], vlan.routes)
|
||||
else:
|
||||
self.assertEqual([], vlan.routes)
|
||||
|
||||
def test__update_vlan_net_data_input_rules(self):
|
||||
ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][
|
||||
'ip_alloc_offset']
|
||||
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
obj.network_subnets = obj._get_network_subnets()
|
||||
obj._update_vlan_net_data(ip_alloc_offset_rules)
|
||||
|
||||
@ -197,12 +373,32 @@ class TestProcessDataSource(unittest.TestCase):
|
||||
self.assertEqual([], vlan.routes)
|
||||
|
||||
def test_load_extracted_data_from_data_source(self):
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
self.assertEqual(self.site_document_data, obj.data)
|
||||
|
||||
def test_load_extracted_data_from_data_source_input_rules(self):
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
self.assertEqual(self.site_document_data, obj.data)
|
||||
|
||||
@mock.patch('yaml.dump', return_value='success')
|
||||
def test_dump_intermediary_file(self, mock_dump):
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
mock_open = mock.mock_open()
|
||||
with mock.patch('spyglass.parser.engine.open', mock_open):
|
||||
obj.dump_intermediary_file(None)
|
||||
mock_dump.assert_called_once_with(
|
||||
self.site_document_data.dict_from_class(),
|
||||
default_flow_style=False)
|
||||
mock_open.return_value.write.assert_called_once()
|
||||
mock_open.return_value.close.assert_called_once()
|
||||
|
||||
@mock.patch('yaml.dump', return_value='success')
|
||||
def test_dump_intermediary_file_input_rules(self, mock_dump):
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
mock_open = mock.mock_open()
|
||||
with mock.patch('spyglass.parser.engine.open', mock_open):
|
||||
obj.dump_intermediary_file(None)
|
||||
@ -216,7 +412,19 @@ class TestProcessDataSource(unittest.TestCase):
|
||||
@mock.patch.object(ProcessDataSource, '_get_genesis_node_details')
|
||||
def test_generate_intermediary_yaml(
|
||||
self, mock__apply_design_rules, mock__get_genesis_node_details):
|
||||
obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
|
||||
result = obj.generate_intermediary_yaml()
|
||||
self.assertEqual(self.site_document_data, result)
|
||||
mock__apply_design_rules.assert_called_once()
|
||||
mock__get_genesis_node_details.assert_called_once()
|
||||
|
||||
@mock.patch.object(ProcessDataSource, '_apply_design_rules')
|
||||
@mock.patch.object(ProcessDataSource, '_get_genesis_node_details')
|
||||
def test_generate_intermediary_yaml_input_rules(
|
||||
self, mock__apply_design_rules, mock__get_genesis_node_details):
|
||||
obj = ProcessDataSource(
|
||||
self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
|
||||
result = obj.generate_intermediary_yaml()
|
||||
self.assertEqual(self.site_document_data, result)
|
||||
mock__apply_design_rules.assert_called_once()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user