Allow rules to be input by user

New tag -r added so that a rules yaml file can be input.
Rules file outlines rules for data manipulation in the engine.
Preexisting rules left in as the default.

Change-Id: Ide8af31b018b4f888486ae6d48ffb441bf9634a7

doc/source/cli.rst

@@ -72,6 +72,11 @@ engineering Excel files. Must be a readable file in YAML format.
 
 Path to site specific configuration YAML. Must be a readable file.
 
+**-r / \\-\\-rule-configuration** (Optional).
+
+Path to rules configuration YAML file. This file defines the rules used for
+data manipulation. Default rules are used if no rules YAML is entered.
+
 **\\-\\-intermediary-schema** (Optional).
 
 Path to the intermediary schema to be used for validation.
@@ -123,6 +128,11 @@ engineering Excel files. Must be a readable file in YAML format.
 
 Path to site specific configuration YAML. Must be a readable file.
 
+**-r / \\-\\-rule-configuration** (Optional).
+
+Path to rules configuration YAML file. This file defines the rules used for
+data manipulation. Default rules are used if no rules YAML is entered.
+
 **\\-\\-intermediary-schema** (Optional).
 
 Path to the intermediary schema to be used for validation.

spyglass/cli.py

@@ -42,6 +42,14 @@ SITE_CONFIGURATION_FILE_OPTION = click.option(
     required=False,
     help='Path to site specific configuration details YAML file.')
 
+RULE_CONFIGURATION_FILE_OPTION = click.option(
+    '-r',
+    '--rule-configuration',
+    'rule_configuration',
+    type=click.Path(exists=True, readable=True, dir_okay=False),
+    required=False,
+    help='Path to data manipulation configuration rules YAML file.')
+
 INTERMEDIARY_DIR_OPTION = click.option(
     '-d',
     '--intermediary-dir',
@@ -151,6 +159,7 @@ def intermediary_processor(plugin_type, **kwargs):
     LOG.info("Apply design rules to the extracted data")
     process_input_ob = ProcessDataSource(
         kwargs['site_name'], data_extractor.data,
+        kwargs.get('rule_configuration', None),
         kwargs.get('intermediary_schema', None),
         kwargs.get('no_validation', False))
     return process_input_ob

spyglass/examples/rules.yaml

@@ -0,0 +1,38 @@
+###########################
+# Global Rules            #
+###########################
+#Rule1:  ip_alloc_offset
+#        Specifies the number of ip addresses to offset from
+#        the start of subnet allocation pool while allocating it to host.
+#        -for vlan it is set to 12 as default.
+#        -for oob it is 10
+#        -for all gateway ip addresss it is set to 1.
+#        -for ingress vip it is 1
+#        -for static end (non pxe) it is -1( means one but last ip of the pool)
+#        -for dhcp end (pxe only) it is -2( 3rd from the last ip of the pool)
+#Rule2:  host_profile_interfaces.
+#        Specifies the network interfaces type and
+#        and their names for a particular hw profile
+#Rule3: hardware_profile
+#       This specifies the profile details  bases on sitetype.
+#       It specifies the profile name and host type for compute,
+#       controller along with hw type
+---
+rule_ip_alloc_offset:
+  name: ip_alloc_offset
+  ip_alloc_offset:
+    default: 12
+    oob: 10
+    gateway: 1
+    ingress_vip: 1
+    static_ip_end: -2
+    dhcp_ip_end: -2
+rule_hardware_profile:
+  name: hardware_profile
+  hardware_profile:
+    foundry:
+      profile_name:
+        compute: dp-r720
+        ctrl: cp-r720
+      hw_type: dell_r720
+...

spyglass/parser/engine.py

@@ -32,6 +32,7 @@ class ProcessDataSource(object):
             self,
             region,
             extracted_data,
+            rules_config,
             intermediary_schema=None,
             no_validation=True):
         # Initialize intermediary and save site type
@@ -40,6 +41,7 @@ class ProcessDataSource(object):
         self.genesis_node = None
         self.network_subnets = None
         self.region_name = region
+        self.rules = rules_config
         self.no_validation = no_validation
         if intermediary_schema and not self.no_validation:
             with open(intermediary_schema, 'r') as loaded_schema:
@@ -110,14 +112,16 @@ class ProcessDataSource(object):
         information. The method calls corresponding rule handler function
         based on rule name and applies them to appropriate data objects.
         """
-
-        LOG.info("Apply design rules")
         # TODO(ian-pittwood): We may want to let users specify these in cli
         #                     opts. We also need better guidelines over how
         #                     to write these rules and how they are applied.
-
+        if self.rules is None:
-        rules_dir = resource_filename("spyglass", "config/")
+            LOG.info("Apply design rules: Default")
-        rules_file = os.path.join(rules_dir, "rules.yaml")
+            rules_dir = resource_filename("spyglass", "config/")
+            rules_file = os.path.join(rules_dir, "rules.yaml")
+        else:
+            LOG.info("Apply design rules: " + str(self.rules))
+            rules_file = self.rules
         rules_data_raw = self._read_file(rules_file)
         rules_yaml = yaml.safe_load(rules_data_raw)
         for rule in rules_yaml.keys():

tests/unit/parser/test_engine.py

@@ -32,10 +32,24 @@ FIXTURE_DIR = os.path.join(
 @mark.usefixtures('rules_data')
 class TestProcessDataSource(unittest.TestCase):
     REGION_NAME = 'test'
+    DEFAULT_RULES = None
+    INPUT_RULES = os.path.join(FIXTURE_DIR, 'rules.yaml')
 
     def test___init__(self):
         expected_data = 'data'
-        obj = ProcessDataSource(self.REGION_NAME, expected_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, expected_data, self.DEFAULT_RULES)
+        self.assertEqual(self.REGION_NAME, obj.region_name)
+        self.assertDictEqual({}, obj.host_type)
+        self.assertEqual(expected_data, obj.data)
+        self.assertIsNone(obj.sitetype)
+        self.assertIsNone(obj.genesis_node)
+        self.assertIsNone(obj.network_subnets)
+
+    def test___init__rules_input(self):
+        expected_data = 'data'
+        obj = ProcessDataSource(
+            self.REGION_NAME, expected_data, self.INPUT_RULES)
         self.assertEqual(self.REGION_NAME, obj.region_name)
         self.assertDictEqual({}, obj.host_type)
         self.assertEqual(expected_data, obj.data)
@@ -59,7 +73,22 @@ class TestProcessDataSource(unittest.TestCase):
             'pxe': IPNetwork('30.30.4.0/25'),
             'storage': IPNetwork('30.31.1.0/25')
         }
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        result = obj._get_network_subnets()
+        self.assertDictEqual(expected_result, result)
+
+    def test__get_network_subnets_input_rules(self):
+        expected_result = {
+            'calico': IPNetwork('30.29.1.0/25'),
+            'oam': IPNetwork('10.0.220.0/26'),
+            'oob': IPNetwork('10.0.220.128/27'),
+            'overlay': IPNetwork('30.19.0.0/25'),
+            'pxe': IPNetwork('30.30.4.0/25'),
+            'storage': IPNetwork('30.31.1.0/25')
+        }
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         result = obj._get_network_subnets()
         self.assertDictEqual(expected_result, result)
 
@@ -67,22 +96,49 @@ class TestProcessDataSource(unittest.TestCase):
         expected_result = self.site_document_data.get_baremetal_host_by_type(
             'genesis')[0]
 
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        obj._get_genesis_node_details()
+        self.assertEqual(expected_result, obj.genesis_node)
+
+    def test__get_genesis_node_details_input_rules(self):
+        expected_result = self.site_document_data.get_baremetal_host_by_type(
+            'genesis')[0]
+
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         obj._get_genesis_node_details()
         self.assertEqual(expected_result, obj.genesis_node)
 
     def test__validate_intermediary_data(self):
         schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
         obj = ProcessDataSource(
-            self.REGION_NAME, self.site_document_data, schema_path, False)
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES,
+            schema_path, False)
+        result = obj._validate_intermediary_data()
+        self.assertIsNone(result)
+
+    def test__validate_intermediary_data_input_rules(self):
+        schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES,
+            schema_path, False)
         result = obj._validate_intermediary_data()
         self.assertIsNone(result)
 
     def test__validate_intermediary_data_invalid(self):
         schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
         obj = ProcessDataSource(
-            self.REGION_NAME, self.invalid_site_document_data, schema_path,
+            self.REGION_NAME, self.invalid_site_document_data,
-            False)
+            self.DEFAULT_RULES, schema_path, False)
+        with self.assertRaises(exceptions.IntermediaryValidationException):
+            obj._validate_intermediary_data()
+
+    def test__validate_intermediary_data_invalid_input_rules(self):
+        schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json')
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.invalid_site_document_data,
+            self.INPUT_RULES, schema_path, False)
         with self.assertRaises(exceptions.IntermediaryValidationException):
             obj._validate_intermediary_data()
 
@@ -90,7 +146,18 @@ class TestProcessDataSource(unittest.TestCase):
     @mock.patch.object(ProcessDataSource, '_apply_rule_hardware_profile')
     def test__apply_design_rules(
             self, mock_rule_hw_profile, mock_rule_ip_alloc_offset):
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        obj._apply_design_rules()
+        mock_rule_hw_profile.assert_called_once()
+        mock_rule_ip_alloc_offset.assert_called_once()
+
+    @mock.patch.object(ProcessDataSource, '_apply_rule_ip_alloc_offset')
+    @mock.patch.object(ProcessDataSource, '_apply_rule_hardware_profile')
+    def test__apply_design_rules_input_rules(
+            self, mock_rule_hw_profile, mock_rule_ip_alloc_offset):
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         obj._apply_design_rules()
         mock_rule_hw_profile.assert_called_once()
         mock_rule_ip_alloc_offset.assert_called_once()
@@ -99,7 +166,28 @@ class TestProcessDataSource(unittest.TestCase):
         input_rules = self.rules_data['rule_hardware_profile'][
             'hardware_profile']
 
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        obj._apply_rule_hardware_profile(input_rules)
+        self.assertEqual(
+            1, len(obj.data.get_baremetal_host_by_type('genesis')))
+        self.assertEqual(
+            3, len(obj.data.get_baremetal_host_by_type('controller')))
+        self.assertEqual(
+            8, len(obj.data.get_baremetal_host_by_type('compute')))
+        for host in obj.data.get_baremetal_host_by_type('genesis'):
+            self.assertEqual('cp-r720', host.host_profile)
+        for host in obj.data.get_baremetal_host_by_type('controller'):
+            self.assertEqual('cp-r720', host.host_profile)
+        for host in obj.data.get_baremetal_host_by_type('compute'):
+            self.assertEqual('dp-r720', host.host_profile)
+
+    def test__apply_rule_hardware_profile_input_rules(self):
+        input_rules = self.rules_data['rule_hardware_profile'][
+            'hardware_profile']
+
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         obj._apply_rule_hardware_profile(input_rules)
         self.assertEqual(
             1, len(obj.data.get_baremetal_host_by_type('genesis')))
@@ -121,7 +209,24 @@ class TestProcessDataSource(unittest.TestCase):
     def test__apply_rule_ip_alloc_offset(
             self, mock__get_network_subnets, mock__update_vlan_net_data,
             mock__update_baremetal_host_ip_data):
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        obj._apply_rule_ip_alloc_offset(self.rules_data)
+        self.assertEqual('success', obj.network_subnets)
+        mock__get_network_subnets.assert_called_once()
+        mock__update_vlan_net_data.assert_called_once_with(self.rules_data)
+        mock__update_baremetal_host_ip_data.assert_called_once_with(
+            self.rules_data)
+
+    @mock.patch.object(ProcessDataSource, '_update_baremetal_host_ip_data')
+    @mock.patch.object(ProcessDataSource, '_update_vlan_net_data')
+    @mock.patch.object(
+        ProcessDataSource, '_get_network_subnets', return_value='success')
+    def test__apply_rule_ip_alloc_offset_input_rules(
+            self, mock__get_network_subnets, mock__update_vlan_net_data,
+            mock__update_baremetal_host_ip_data):
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         obj._apply_rule_ip_alloc_offset(self.rules_data)
         self.assertEqual('success', obj.network_subnets)
         mock__get_network_subnets.assert_called_once()
@@ -130,7 +235,26 @@ class TestProcessDataSource(unittest.TestCase):
             self.rules_data)
 
     def test__update_baremetal_host_ip_data(self):
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        obj.network_subnets = obj._get_network_subnets()
+        ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][
+            'ip_alloc_offset']
+        obj._update_baremetal_host_ip_data(ip_alloc_offset_rules)
+
+        counter = 0
+        for rack in obj.data.baremetal:
+            for host in rack.hosts:
+                for net_type, net_ip in iter(host.ip):
+                    ips = list(obj.network_subnets[net_type])
+                    self.assertEqual(
+                        str(ips[counter + ip_alloc_offset_rules['default']]),
+                        net_ip)
+                counter += 1
+
+    def test__update_baremetal_host_ip_data_input_rules(self):
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         obj.network_subnets = obj._get_network_subnets()
         ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][
             'ip_alloc_offset']
@@ -150,7 +274,59 @@ class TestProcessDataSource(unittest.TestCase):
         ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][
             'ip_alloc_offset']
 
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        obj.network_subnets = obj._get_network_subnets()
+        obj._update_vlan_net_data(ip_alloc_offset_rules)
+
+        ingress_data = obj.data.network.get_vlan_data_by_name('ingress')
+        subnet = IPNetwork(ingress_data.subnet[0])
+        ips = list(subnet)
+        self.assertEqual(
+            str(ips[ip_alloc_offset_rules['ingress_vip']]),
+            obj.data.network.bgp['ingress_vip'])
+        self.assertEqual(
+            ingress_data.subnet[0],
+            obj.data.network.bgp['public_service_cidr'])
+        subnets = obj.network_subnets
+        for vlan in self.site_document_data.network.vlan_network_data:
+            if vlan.role == 'ingress':
+                continue
+            ips = list(subnets[vlan.role])
+            self.assertEqual(
+                str(ips[ip_alloc_offset_rules['gateway']]), vlan.gateway)
+
+            if vlan.role == 'oob':
+                ip_offset = ip_alloc_offset_rules['oob']
+            else:
+                ip_offset = ip_alloc_offset_rules['default']
+            self.assertEqual(str(ips[1]), vlan.reserved_start)
+            self.assertEqual(str(ips[ip_offset]), vlan.reserved_end)
+            self.assertEqual(str(ips[ip_offset + 1]), vlan.static_start)
+
+            if vlan.role == 'pxe':
+                self.assertEqual(
+                    str(ips[(len(ips) // 2) - 1]), vlan.static_end)
+                self.assertEqual(str(ips[len(ips) // 2]), vlan.dhcp_start)
+                self.assertEqual(
+                    str(ips[ip_alloc_offset_rules['dhcp_ip_end']]),
+                    vlan.dhcp_end)
+            else:
+                self.assertEqual(
+                    str(ips[ip_alloc_offset_rules['static_ip_end']]),
+                    vlan.static_end)
+
+            if vlan.role == 'oam':
+                self.assertEqual(['0.0.0.0/0'], vlan.routes)
+            else:
+                self.assertEqual([], vlan.routes)
+
+    def test__update_vlan_net_data_input_rules(self):
+        ip_alloc_offset_rules = self.rules_data['rule_ip_alloc_offset'][
+            'ip_alloc_offset']
+
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         obj.network_subnets = obj._get_network_subnets()
         obj._update_vlan_net_data(ip_alloc_offset_rules)
 
@@ -197,12 +373,32 @@ class TestProcessDataSource(unittest.TestCase):
                 self.assertEqual([], vlan.routes)
 
     def test_load_extracted_data_from_data_source(self):
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        self.assertEqual(self.site_document_data, obj.data)
+
+    def test_load_extracted_data_from_data_source_input_rules(self):
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         self.assertEqual(self.site_document_data, obj.data)
 
     @mock.patch('yaml.dump', return_value='success')
     def test_dump_intermediary_file(self, mock_dump):
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        mock_open = mock.mock_open()
+        with mock.patch('spyglass.parser.engine.open', mock_open):
+            obj.dump_intermediary_file(None)
+        mock_dump.assert_called_once_with(
+            self.site_document_data.dict_from_class(),
+            default_flow_style=False)
+        mock_open.return_value.write.assert_called_once()
+        mock_open.return_value.close.assert_called_once()
+
+    @mock.patch('yaml.dump', return_value='success')
+    def test_dump_intermediary_file_input_rules(self, mock_dump):
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         mock_open = mock.mock_open()
         with mock.patch('spyglass.parser.engine.open', mock_open):
             obj.dump_intermediary_file(None)
@@ -216,7 +412,19 @@ class TestProcessDataSource(unittest.TestCase):
     @mock.patch.object(ProcessDataSource, '_get_genesis_node_details')
     def test_generate_intermediary_yaml(
             self, mock__apply_design_rules, mock__get_genesis_node_details):
-        obj = ProcessDataSource(self.REGION_NAME, self.site_document_data)
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.DEFAULT_RULES)
+        result = obj.generate_intermediary_yaml()
+        self.assertEqual(self.site_document_data, result)
+        mock__apply_design_rules.assert_called_once()
+        mock__get_genesis_node_details.assert_called_once()
+
+    @mock.patch.object(ProcessDataSource, '_apply_design_rules')
+    @mock.patch.object(ProcessDataSource, '_get_genesis_node_details')
+    def test_generate_intermediary_yaml_input_rules(
+            self, mock__apply_design_rules, mock__get_genesis_node_details):
+        obj = ProcessDataSource(
+            self.REGION_NAME, self.site_document_data, self.INPUT_RULES)
         result = obj.generate_intermediary_yaml()
         self.assertEqual(self.site_document_data, result)
         mock__apply_design_rules.assert_called_once()