diff --git a/doc/source/cli.rst b/doc/source/cli.rst index a59504f..50dba1e 100644 --- a/doc/source/cli.rst +++ b/doc/source/cli.rst @@ -56,6 +56,11 @@ engineering Excel files. Must be a readable file in YAML format. Path to site specific configuration YAML. Must be a readable file. +**-r / \\-\\-rule-configuration** (Optional). + +Path to rules configuration YAML file. This file defines the rules used for +data manipulation. Default rules are used if no rules YAML is entered. + **\\-\\-intermediary-schema** (Optional). Path to the intermediary schema to be used for validation. @@ -107,6 +112,11 @@ engineering Excel files. Must be a readable file in YAML format. Path to site specific configuration YAML. Must be a readable file. +**-r / \\-\\-rule-configuration** (Optional). + +Path to rules configuration YAML file. This file defines the rules used for +data manipulation. Default rules are used if no rules YAML is entered. + **\\-\\-intermediary-schema** (Optional). Path to the intermediary schema to be used for validation. diff --git a/spyglass/cli.py b/spyglass/cli.py index b16ea3e..e618718 100644 --- a/spyglass/cli.py +++ b/spyglass/cli.py @@ -42,6 +42,14 @@ SITE_CONFIGURATION_FILE_OPTION = click.option( required=False, help='Path to site specific configuration details YAML file.') +RULE_CONFIGURATION_FILE_OPTION = click.option( + '-r', + '--rule-configuration', + 'rule_configuration', + type=click.Path(exists=True, readable=True, dir_okay=False), + required=False, + help='Path to data manipulation configuration rules YAML file.') + INTERMEDIARY_DIR_OPTION = click.option( '-d', '--intermediary-dir', @@ -151,6 +159,7 @@ def intermediary_processor(plugin_type, **kwargs): LOG.info("Apply design rules to the extracted data") process_input_ob = ProcessDataSource( kwargs['site_name'], data_extractor.data, + kwargs.get('rule_configuration', None), kwargs.get('intermediary_schema', None), kwargs.get('no_validation', False)) return process_input_ob diff --git a/spyglass/examples/rules.yaml b/spyglass/examples/rules.yaml new file mode 100644 index 0000000..dfe4025 --- /dev/null +++ b/spyglass/examples/rules.yaml @@ -0,0 +1,38 @@ +########################### +# Global Rules # +########################### +#Rule1: ip_alloc_offset +# Specifies the number of ip addresses to offset from +# the start of subnet allocation pool while allocating it to host. +# -for vlan it is set to 12 as default. +# -for oob it is 10 +# -for all gateway ip addresss it is set to 1. +# -for ingress vip it is 1 +# -for static end (non pxe) it is -1( means one but last ip of the pool) +# -for dhcp end (pxe only) it is -2( 3rd from the last ip of the pool) +#Rule2: host_profile_interfaces. +# Specifies the network interfaces type and +# and their names for a particular hw profile +#Rule3: hardware_profile +# This specifies the profile details bases on sitetype. +# It specifies the profile name and host type for compute, +# controller along with hw type +--- +rule_ip_alloc_offset: + name: ip_alloc_offset + ip_alloc_offset: + default: 12 + oob: 10 + gateway: 1 + ingress_vip: 1 + static_ip_end: -2 + dhcp_ip_end: -2 +rule_hardware_profile: + name: hardware_profile + hardware_profile: + foundry: + profile_name: + compute: dp-r720 + ctrl: cp-r720 + hw_type: dell_r720 +... diff --git a/spyglass/parser/engine.py b/spyglass/parser/engine.py index 68df990..0a8a978 100755 --- a/spyglass/parser/engine.py +++ b/spyglass/parser/engine.py @@ -32,6 +32,7 @@ class ProcessDataSource(object): self, region, extracted_data, + rules_config, intermediary_schema=None, no_validation=True): # Initialize intermediary and save site type @@ -40,6 +41,7 @@ class ProcessDataSource(object): self.genesis_node = None self.network_subnets = None self.region_name = region + self.rules = rules_config self.no_validation = no_validation if intermediary_schema and not self.no_validation: with open(intermediary_schema, 'r') as loaded_schema: @@ -110,14 +112,16 @@ class ProcessDataSource(object): information. The method calls corresponding rule handler function based on rule name and applies them to appropriate data objects. """ - - LOG.info("Apply design rules") # TODO(ian-pittwood): We may want to let users specify these in cli # opts. We also need better guidelines over how # to write these rules and how they are applied. - - rules_dir = resource_filename("spyglass", "config/") - rules_file = os.path.join(rules_dir, "rules.yaml") + if self.rules is None: + LOG.info("Apply design rules: Default") + rules_dir = resource_filename("spyglass", "config/") + rules_file = os.path.join(rules_dir, "rules.yaml") + else: + LOG.info("Apply design rules: " + str(self.rules)) + rules_file = self.rules rules_data_raw = self._read_file(rules_file) rules_yaml = yaml.safe_load(rules_data_raw) for rule in rules_yaml.keys(): diff --git a/tests/unit/parser/test_engine.py b/tests/unit/parser/test_engine.py index b6e120f..b7faffb 100644 --- a/tests/unit/parser/test_engine.py +++ b/tests/unit/parser/test_engine.py @@ -32,10 +32,24 @@ FIXTURE_DIR = os.path.join( @mark.usefixtures('rules_data') class TestProcessDataSource(unittest.TestCase): REGION_NAME = 'test' + DEFAULT_RULES = None + INPUT_RULES = os.path.join(FIXTURE_DIR, 'rules.yaml') def test___init__(self): expected_data = 'data' - obj = ProcessDataSource(self.REGION_NAME, expected_data) + obj = ProcessDataSource( + self.REGION_NAME, expected_data, self.DEFAULT_RULES) + self.assertEqual(self.REGION_NAME, obj.region_name) + self.assertDictEqual({}, obj.host_type) + self.assertEqual(expected_data, obj.data) + self.assertIsNone(obj.sitetype) + self.assertIsNone(obj.genesis_node) + self.assertIsNone(obj.network_subnets) + + def test___init__rules_input(self): + expected_data = 'data' + obj = ProcessDataSource( + self.REGION_NAME, expected_data, self.INPUT_RULES) self.assertEqual(self.REGION_NAME, obj.region_name) self.assertDictEqual({}, obj.host_type) self.assertEqual(expected_data, obj.data) @@ -59,7 +73,22 @@ class TestProcessDataSource(unittest.TestCase): 'pxe': IPNetwork('30.30.4.0/25'), 'storage': IPNetwork('30.31.1.0/25') } - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + result = obj._get_network_subnets() + self.assertDictEqual(expected_result, result) + + def test__get_network_subnets_input_rules(self): + expected_result = { + 'calico': IPNetwork('30.29.1.0/25'), + 'oam': IPNetwork('10.0.220.0/26'), + 'oob': IPNetwork('10.0.220.128/27'), + 'overlay': IPNetwork('30.19.0.0/25'), + 'pxe': IPNetwork('30.30.4.0/25'), + 'storage': IPNetwork('30.31.1.0/25') + } + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) result = obj._get_network_subnets() self.assertDictEqual(expected_result, result) @@ -67,22 +96,49 @@ class TestProcessDataSource(unittest.TestCase): expected_result = self.site_document_data.get_baremetal_host_by_type( 'genesis')[0] - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + obj._get_genesis_node_details() + self.assertEqual(expected_result, obj.genesis_node) + + def test__get_genesis_node_details_input_rules(self): + expected_result = self.site_document_data.get_baremetal_host_by_type( + 'genesis')[0] + + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) obj._get_genesis_node_details() self.assertEqual(expected_result, obj.genesis_node) def test__validate_intermediary_data(self): schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json') obj = ProcessDataSource( - self.REGION_NAME, self.site_document_data, schema_path, False) + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES, + schema_path, False) + result = obj._validate_intermediary_data() + self.assertIsNone(result) + + def test__validate_intermediary_data_input_rules(self): + schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json') + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES, + schema_path, False) result = obj._validate_intermediary_data() self.assertIsNone(result) def test__validate_intermediary_data_invalid(self): schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json') obj = ProcessDataSource( - self.REGION_NAME, self.invalid_site_document_data, schema_path, - False) + self.REGION_NAME, self.invalid_site_document_data, + self.DEFAULT_RULES, schema_path, False) + with self.assertRaises(exceptions.IntermediaryValidationException): + obj._validate_intermediary_data() + + def test__validate_intermediary_data_invalid_input_rules(self): + schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json') + obj = ProcessDataSource( + self.REGION_NAME, self.invalid_site_document_data, + self.INPUT_RULES, schema_path, False) with self.assertRaises(exceptions.IntermediaryValidationException): obj._validate_intermediary_data() @@ -90,7 +146,18 @@ class TestProcessDataSource(unittest.TestCase): @mock.patch.object(ProcessDataSource, '_apply_rule_hardware_profile') def test__apply_design_rules( self, mock_rule_hw_profile, mock_rule_ip_alloc_offset): - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + obj._apply_design_rules() + mock_rule_hw_profile.assert_called_once() + mock_rule_ip_alloc_offset.assert_called_once() + + @mock.patch.object(ProcessDataSource, '_apply_rule_ip_alloc_offset') + @mock.patch.object(ProcessDataSource, '_apply_rule_hardware_profile') + def test__apply_design_rules_input_rules( + self, mock_rule_hw_profile, mock_rule_ip_alloc_offset): + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) obj._apply_design_rules() mock_rule_hw_profile.assert_called_once() mock_rule_ip_alloc_offset.assert_called_once() @@ -99,7 +166,28 @@ class TestProcessDataSource(unittest.TestCase): input_rules = self.rules_data['rule_hardware_profile'][ 'hardware_profile'] - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + obj._apply_rule_hardware_profile(input_rules) + self.assertEqual( + 1, len(obj.data.get_baremetal_host_by_type('genesis'))) + self.assertEqual( + 3, len(obj.data.get_baremetal_host_by_type('controller'))) + self.assertEqual( + 8, len(obj.data.get_baremetal_host_by_type('compute'))) + for host in obj.data.get_baremetal_host_by_type('genesis'): + self.assertEqual('cp-r720', host.host_profile) + for host in obj.data.get_baremetal_host_by_type('controller'): + self.assertEqual('cp-r720', host.host_profile) + for host in obj.data.get_baremetal_host_by_type('compute'): + self.assertEqual('dp-r720', host.host_profile) + + def test__apply_rule_hardware_profile_input_rules(self): + input_rules = self.rules_data['rule_hardware_profile'][ + 'hardware_profile'] + + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) obj._apply_rule_hardware_profile(input_rules) self.assertEqual( 1, len(obj.data.get_baremetal_host_by_type('genesis'))) @@ -121,7 +209,24 @@ class TestProcessDataSource(unittest.TestCase): def test__apply_rule_ip_alloc_offset( self, mock__get_network_subnets, mock__update_vlan_net_data, mock__update_baremetal_host_ip_data): - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + obj._apply_rule_ip_alloc_offset(self.rules_data) + self.assertEqual('success', obj.network_subnets) + mock__get_network_subnets.assert_called_once() + mock__update_vlan_net_data.assert_called_once_with(self.rules_data) + mock__update_baremetal_host_ip_data.assert_called_once_with( + self.rules_data) + + @mock.patch.object(ProcessDataSource, '_update_baremetal_host_ip_data') + @mock.patch.object(ProcessDataSource, '_update_vlan_net_data') + @mock.patch.object( + ProcessDataSource, '_get_network_subnets', return_value='success') + def test__apply_rule_ip_alloc_offset_input_rules( + self, mock__get_network_subnets, mock__update_vlan_net_data, + mock__update_baremetal_host_ip_data): + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) obj._apply_rule_ip_alloc_offset(self.rules_data) self.assertEqual('success', obj.network_subnets) mock__get_network_subnets.assert_called_once() @@ -130,7 +235,26 @@ class TestProcessDataSource(unittest.TestCase): self.rules_data) def test__update_baremetal_host_ip_data(self): - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + obj.network_subnets = obj._get_network_subnets() + ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][ + 'ip_alloc_offset'] + obj._update_baremetal_host_ip_data(ip_alloc_offset_rules) + + counter = 0 + for rack in obj.data.baremetal: + for host in rack.hosts: + for net_type, net_ip in iter(host.ip): + ips = list(obj.network_subnets[net_type]) + self.assertEqual( + str(ips[counter + ip_alloc_offset_rules['default']]), + net_ip) + counter += 1 + + def test__update_baremetal_host_ip_data_input_rules(self): + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) obj.network_subnets = obj._get_network_subnets() ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][ 'ip_alloc_offset'] @@ -150,7 +274,59 @@ class TestProcessDataSource(unittest.TestCase): ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][ 'ip_alloc_offset'] - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + obj.network_subnets = obj._get_network_subnets() + obj._update_vlan_net_data(ip_alloc_offset_rules) + + ingress_data = obj.data.network.get_vlan_data_by_name('ingress') + subnet = IPNetwork(ingress_data.subnet[0]) + ips = list(subnet) + self.assertEqual( + str(ips[ip_alloc_offset_rules['ingress_vip']]), + obj.data.network.bgp['ingress_vip']) + self.assertEqual( + ingress_data.subnet[0], + obj.data.network.bgp['public_service_cidr']) + subnets = obj.network_subnets + for vlan in self.site_document_data.network.vlan_network_data: + if vlan.role == 'ingress': + continue + ips = list(subnets[vlan.role]) + self.assertEqual( + str(ips[ip_alloc_offset_rules['gateway']]), vlan.gateway) + + if vlan.role == 'oob': + ip_offset = ip_alloc_offset_rules['oob'] + else: + ip_offset = ip_alloc_offset_rules['default'] + self.assertEqual(str(ips[1]), vlan.reserved_start) + self.assertEqual(str(ips[ip_offset]), vlan.reserved_end) + self.assertEqual(str(ips[ip_offset + 1]), vlan.static_start) + + if vlan.role == 'pxe': + self.assertEqual( + str(ips[(len(ips) // 2) - 1]), vlan.static_end) + self.assertEqual(str(ips[len(ips) // 2]), vlan.dhcp_start) + self.assertEqual( + str(ips[ip_alloc_offset_rules['dhcp_ip_end']]), + vlan.dhcp_end) + else: + self.assertEqual( + str(ips[ip_alloc_offset_rules['static_ip_end']]), + vlan.static_end) + + if vlan.role == 'oam': + self.assertEqual(['0.0.0.0/0'], vlan.routes) + else: + self.assertEqual([], vlan.routes) + + def test__update_vlan_net_data_input_rules(self): + ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][ + 'ip_alloc_offset'] + + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) obj.network_subnets = obj._get_network_subnets() obj._update_vlan_net_data(ip_alloc_offset_rules) @@ -197,12 +373,32 @@ class TestProcessDataSource(unittest.TestCase): self.assertEqual([], vlan.routes) def test_load_extracted_data_from_data_source(self): - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + self.assertEqual(self.site_document_data, obj.data) + + def test_load_extracted_data_from_data_source_input_rules(self): + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) self.assertEqual(self.site_document_data, obj.data) @mock.patch('yaml.dump', return_value='success') def test_dump_intermediary_file(self, mock_dump): - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + mock_open = mock.mock_open() + with mock.patch('spyglass.parser.engine.open', mock_open): + obj.dump_intermediary_file(None) + mock_dump.assert_called_once_with( + self.site_document_data.dict_from_class(), + default_flow_style=False) + mock_open.return_value.write.assert_called_once() + mock_open.return_value.close.assert_called_once() + + @mock.patch('yaml.dump', return_value='success') + def test_dump_intermediary_file_input_rules(self, mock_dump): + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) mock_open = mock.mock_open() with mock.patch('spyglass.parser.engine.open', mock_open): obj.dump_intermediary_file(None) @@ -216,7 +412,19 @@ class TestProcessDataSource(unittest.TestCase): @mock.patch.object(ProcessDataSource, '_get_genesis_node_details') def test_generate_intermediary_yaml( self, mock__apply_design_rules, mock__get_genesis_node_details): - obj = ProcessDataSource(self.REGION_NAME, self.site_document_data) + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES) + result = obj.generate_intermediary_yaml() + self.assertEqual(self.site_document_data, result) + mock__apply_design_rules.assert_called_once() + mock__get_genesis_node_details.assert_called_once() + + @mock.patch.object(ProcessDataSource, '_apply_design_rules') + @mock.patch.object(ProcessDataSource, '_get_genesis_node_details') + def test_generate_intermediary_yaml_input_rules( + self, mock__apply_design_rules, mock__get_genesis_node_details): + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, self.INPUT_RULES) result = obj.generate_intermediary_yaml() self.assertEqual(self.site_document_data, result) mock__apply_design_rules.assert_called_once()