Testing for plugin

Adds tests for the Excel plugin's parser and extractor.

Enables pep8 and fmt checks on the tests directory.

Increases plugin test coverage to 94%, sets new minimum to 92%.

Fixes DNS and NTP server extraction with regex.

Updates file licenses.

Change-Id: I35ee97574e6d63b7a82cfa94caf79db5db9755e7

spyglass_plugin_xls/check_exceptions.py

@@ -1,4 +1,4 @@
-# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,3 +36,17 @@ class NoSpecMatched(BaseError):
         print(
             "No spec matched. Following are the available specs:\n".format(
                 self.specs))
+
+
+class ExcelFileNotSpecified(BaseError):
+
+    @staticmethod
+    def display_error():
+        print("Engineering excel file not specified")
+
+
+class ExcelSpecNotSpecified(BaseError):
+
+    @staticmethod
+    def display_error():
+        print("Engineering excel spec not specified")

spyglass_plugin_xls/cli.py

@@ -1,4 +1,4 @@
-# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

spyglass_plugin_xls/examples/excel_spec.yaml

@@ -1,5 +1,4 @@
-# Copyright 2018 The Openstack-Helm Authors.
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
-# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

spyglass_plugin_xls/examples/site_config.yaml

@@ -1,3 +1,16 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 ##############################################
 # Site Specific Spyglass XLS Plugin Settings #
 ##############################################

spyglass_plugin_xls/excel.py

@@ -1,4 +1,4 @@
-# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +18,9 @@ import pprint
 import re
 from spyglass.data_extractor.base import BaseDataSourcePlugin
 from spyglass.data_extractor import models
+
+from spyglass_plugin_xls.check_exceptions import ExcelFileNotSpecified
+from spyglass_plugin_xls.check_exceptions import ExcelSpecNotSpecified
 from spyglass_plugin_xls.excel_parser import ExcelParser
 
 LOG = logging.getLogger(__name__)
@@ -66,13 +69,11 @@ class ExcelPlugin(BaseDataSourcePlugin):
         and excel specs are not specified. The below code has been
         written as an additional safeguard.
         """
-        if not kwargs["excel_file"]:
+        if 'excel_file' not in kwargs:
-            LOG.error("Engineering excel file not specified: Spyglass exited!")
+            raise ExcelFileNotSpecified()
-            exit()
         excel_file_info = kwargs["excel_file"]
-        if not kwargs["excel_spec"]:
+        if 'excel_spec' not in kwargs:
-            LOG.error("Engineering spec file not specified: Spyglass exited!")
+            raise ExcelSpecNotSpecified()
-            exit()
         excel_spec_info = kwargs["excel_spec"]
         plugin_conf = {
             "excel_path": excel_file_info,
@@ -161,7 +162,7 @@ class ExcelPlugin(BaseDataSourcePlugin):
             vlan_list.append(models.VLANNetworkData(**tmp_vlan))
         return vlan_list
 
-    def get_ips(self, region, host=None):
+    def get_ips(self, region, host):
         """Return list of IPs on the host
 
         :param string region: Region name
@@ -255,7 +256,7 @@ class ExcelPlugin(BaseDataSourcePlugin):
         data_dict['dns'] = self.get_dns_servers(region)
         data_dict['ntp'] = self.get_ntp_servers(region)
         data_dict['ldap'] = self.get_ldap_information(region)
-        return models.SiteInfo(region, **data_dict)
+        return models.SiteInfo(region_name=region, **data_dict)
 
     def _get_excel_obj(self):
         """Creation of an ExcelParser object to store site information.
@@ -271,7 +272,8 @@ class ExcelPlugin(BaseDataSourcePlugin):
         """Extracts raw information from excel file based on excel spec"""
         self.parsed_xl_data = self.excel_obj.get_data()
 
-    def _get_network_name_from_vlan_name(self, vlan_name):
+    @staticmethod
+    def _get_network_name_from_vlan_name(vlan_name):
         """Network names are ksn, oam, oob, overlay, storage, pxe
 
 
@@ -317,7 +319,8 @@ class ExcelPlugin(BaseDataSourcePlugin):
             "Unable to recognize VLAN name extracted from Plugin data source")
         return ""
 
-    def _get_formatted_server_list(self, server_list):
+    @staticmethod
+    def _get_formatted_server_list(server_list):
         """Format dns and ntp server list as comma separated string"""
 
         # dns/ntp server info from excel is of the format

spyglass_plugin_xls/excel_parser.py

@@ -1,4 +1,4 @@
-# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,16 +30,20 @@ LOG = logging.getLogger(__name__)
 class ExcelParser(object):
     """Parse data from excel into a dict"""
 
-    def __init__(self, file_name, excel_specs):
+    def __init__(self, file_name: str, excel_specs: str):
+        """Initializes an ExcelParser to extract data from the Excel workbook
+
+        :param file_name: path to the Excel workbook
+        :param excel_specs: path to the Excel workbook spec
+        """
         self.file_name = file_name
         with open(excel_specs, "r") as f:
             spec_raw_data = f.read()
         self.excel_specs = yaml.safe_load(spec_raw_data)
         # A combined design spec, returns a workbook object after combining
         # all the inputs excel specs
-        combined_design_spec = self.combine_excel_design_specs(file_name)
+        combined_design_spec = self.load_excel_data(file_name)
         self.wb_combined = combined_design_spec
-        self.filenames = file_name
         self.spec = "xl_spec"
 
     @staticmethod
@@ -75,19 +79,22 @@ class ExcelParser(object):
                         return spec
         raise NoSpecMatched(self.excel_specs)
 
-    def get_ipmi_data(self):
+    def _get_workbook(self):
-        """Read IPMI data from the sheet"""
-
-        ipmi_data = {}
-        hosts = []
         provided_sheetname = self.excel_specs["specs"][
             self.spec]["ipmi_sheet_name"]
         workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
             provided_sheetname)
         if workbook_object is not None:
-            ws = workbook_object[extracted_sheetname]
+            return workbook_object[extracted_sheetname]
         else:
-            ws = self.wb_combined[provided_sheetname]
+            return self.wb_combined[provided_sheetname]
+
+    def get_ipmi_data(self):
+        """Read IPMI data from the sheet"""
+
+        ipmi_data = {}
+        hosts = []
+        ws = self._get_workbook()
         row = self.excel_specs["specs"][self.spec]["start_row"]
         end_row = self.excel_specs["specs"][self.spec]["end_row"]
         hostname_col = self.excel_specs["specs"][self.spec]["hostname_col"]
@@ -119,7 +126,7 @@ class ExcelParser(object):
                             self.spec, row, host_profile_col))
             except RuntimeError as rerror:
                 LOG.critical(rerror)
-                sys.exit("Tugboat exited!!")
+                sys.exit("Spyglass exited")
             ipmi_data[hostname] = {
                 "ipmi_address": ipmi_address,
                 "ipmi_gateway": ipmi_gateway,
@@ -157,14 +164,7 @@ class ExcelParser(object):
     def get_private_network_data(self):
         """Read network data from the private ip sheet"""
 
-        provided_sheetname = self.excel_specs["specs"][
+        ws = self._get_workbook()
-            self.spec]["private_ip_sheet"]
-        workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
-            provided_sheetname)
-        if workbook_object is not None:
-            ws = workbook_object[extracted_sheetname]
-        else:
-            ws = self.wb_combined[provided_sheetname]
         vlan_data = self.get_private_vlan_data(ws)
         network_data = {}
         row = self.excel_specs["specs"][self.spec]["net_start_row"]
@@ -208,14 +208,7 @@ class ExcelParser(object):
         """Read public network data from public ip data"""
 
         network_data = {}
-        provided_sheetname = self.excel_specs["specs"][
+        ws = self._get_workbook()
-            self.spec]["public_ip_sheet"]
-        workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
-            provided_sheetname)
-        if workbook_object is not None:
-            ws = workbook_object[extracted_sheetname]
-        else:
-            ws = self.wb_combined[provided_sheetname]
         oam_row = self.excel_specs["specs"][self.spec]["oam_ip_row"]
         oam_col = self.excel_specs["specs"][self.spec]["oam_ip_col"]
         oam_vlan_col = self.excel_specs["specs"][self.spec]["oam_vlan_col"]
@@ -250,13 +243,8 @@ class ExcelParser(object):
 
         site_info = {}
         provided_sheetname = self.excel_specs["specs"][
-            self.spec]["dns_ntp_ldap_sheet"]
+            self.spec]["ipmi_sheet_name"]
-        workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
+        ws = self._get_workbook()
-            provided_sheetname)
-        if workbook_object is not None:
-            ws = workbook_object[extracted_sheetname]
-        else:
-            ws = self.wb_combined[provided_sheetname]
         dns_row = self.excel_specs["specs"][self.spec]["dns_row"]
         dns_col = self.excel_specs["specs"][self.spec]["dns_col"]
         ntp_row = self.excel_specs["specs"][self.spec]["ntp_row"]
@@ -282,17 +270,8 @@ class ExcelParser(object):
         except RuntimeError as rerror:
             LOG.critical(rerror)
             sys.exit("Tugboat exited!!")
-
+        dns_servers = list(filter(None, re.split(" |,|\n", dns_servers)))
-        dns_servers = dns_servers.replace("\n", " ")
+        ntp_servers = list(filter(None, re.split(" |,|\n", ntp_servers)))
-        ntp_servers = ntp_servers.replace("\n", " ")
-        if "," in dns_servers:
-            dns_servers = dns_servers.split(",")
-        else:
-            dns_servers = dns_servers.split()
-        if "," in ntp_servers:
-            ntp_servers = ntp_servers.split(",")
-        else:
-            ntp_servers = ntp_servers.split()
         site_info = {
             "location": self.get_location_data(),
             "dns": dns_servers,
@@ -316,14 +295,7 @@ class ExcelParser(object):
     def get_location_data(self):
         """Read location data from the site and zone sheet"""
 
-        provided_sheetname = self.excel_specs["specs"][
+        ws = self._get_workbook()
-            self.spec]["location_sheet"]
-        workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
-            provided_sheetname)
-        if workbook_object is not None:
-            ws = workbook_object[extracted_sheetname]
-        else:
-            ws = self.wb_combined[provided_sheetname]
         corridor_row = self.excel_specs["specs"][self.spec]["corridor_row"]
         column = self.excel_specs["specs"][self.spec]["column"]
         site_name_row = self.excel_specs["specs"][self.spec]["site_name_row"]
@@ -356,22 +328,18 @@ class ExcelParser(object):
         sheet_name_list.append(dns_ntp_ldap_sheet_name)
         location_sheet_name = spec_item["location_sheet"]
         sheet_name_list.append(location_sheet_name)
-        try:
+        for sheetname in sheet_name_list:
-            for sheetname in sheet_name_list:
+            workbook_object, extracted_sheetname = (
-                workbook_object, extracted_sheetname = (
+                self.get_xl_obj_and_sheetname(sheetname))
-                    self.get_xl_obj_and_sheetname(sheetname))
+            if workbook_object is not None:
-                if workbook_object is not None:
+                wb = workbook_object
-                    wb = workbook_object
+                sheetname = extracted_sheetname
-                    sheetname = extracted_sheetname
+            else:
-                else:
+                wb = self.wb_combined
-                    wb = self.wb_combined
 
-                if sheetname not in wb.sheetnames:
+            if sheetname not in wb.sheetnames:
-                    raise RuntimeError(
+                raise RuntimeError(
-                        "SheetName '{}' not found ".format(sheetname))
+                    "SheetName '{}' not found ".format(sheetname))
-        except RuntimeError as rerror:
-            LOG.critical(rerror)
-            sys.exit("Tugboat exited!!")
 
         LOG.info("Sheet names in excel spec validated")
 
@@ -398,29 +366,29 @@ class ExcelParser(object):
         )
         return data
 
-    def combine_excel_design_specs(self, filenames):
+    @staticmethod
+    def load_excel_data(filename):
         """Combines multiple excel file to a single design spec"""
 
         design_spec = Workbook()
-        for exel_file in filenames:
+        loaded_workbook = load_workbook(filename, data_only=True)
-            loaded_workbook = load_workbook(exel_file, data_only=True)
+        for names in loaded_workbook.sheetnames:
-            for names in loaded_workbook.sheetnames:
+            design_spec_worksheet = design_spec.create_sheet(names)
-                design_spec_worksheet = design_spec.create_sheet(names)
+            loaded_workbook_ws = loaded_workbook[names]
-                loaded_workbook_ws = loaded_workbook[names]
+            for row in loaded_workbook_ws:
-                for row in loaded_workbook_ws:
+                for cell in row:
-                    for cell in row:
+                    design_spec_worksheet[cell.coordinate].value = cell.value
-                        design_spec_worksheet[
-                            cell.coordinate].value = cell.value
         return design_spec
 
-    def get_xl_obj_and_sheetname(self, sheetname):
+    @staticmethod
+    def get_xl_obj_and_sheetname(sheetname):
         """The logic confirms if the sheetname is specified for example as:
 
         'MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs'
         """
 
         if re.search(".xlsx", sheetname) or re.search(".xls", sheetname):
-            """ Extract file name """
+            # Extract file name
             source_xl_file = sheetname.split(":")[0]
             wb = load_workbook(source_xl_file, data_only=True)
             return [wb, sheetname.split(":")[1]]

tests/conftest.py

@@ -0,0 +1,159 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+
+@pytest.fixture(scope='class')
+def site_data(request):
+    request.cls.site_data = {
+        'ipmi_data': [
+            {
+                'cab2r72c12': {
+                    'ipmi_address': '10.0.220.138',
+                    'ipmi_gateway': '10.0.220.129',
+                    'host_profile': 'dp-r720'
+                },
+                'cab2r72c13': {
+                    'ipmi_address': '10.0.220.139',
+                    'ipmi_gateway': '10.0.220.129',
+                    'host_profile': 'dp-r720'
+                },
+                'cab2r72c14': {
+                    'ipmi_address': '10.0.220.140',
+                    'ipmi_gateway': '10.0.220.129',
+                    'host_profile': 'dp-r720'
+                },
+                'cab2r72c15': {
+                    'ipmi_address': '10.0.220.141',
+                    'ipmi_gateway': '10.0.220.129',
+                    'host_profile': 'dp-r720'
+                },
+                'cab2r72c16': {
+                    'ipmi_address': '10.0.220.142',
+                    'ipmi_gateway': '10.0.220.129',
+                    'host_profile': 'cp-r720'
+                },
+                'cab2r72c17': {
+                    'ipmi_address': '10.0.220.143',
+                    'ipmi_gateway': '10.0.220.129',
+                    'host_profile': 'cp-r720'
+                },
+                'cab2r73c12': {
+                    'ipmi_address': '10.0.220.170',
+                    'ipmi_gateway': '10.0.220.161',
+                    'host_profile': 'dp-r720'
+                },
+                'cab2r73c13': {
+                    'ipmi_address': '10.0.220.171',
+                    'ipmi_gateway': '10.0.220.161',
+                    'host_profile': 'dp-r720'
+                },
+                'cab2r73c14': {
+                    'ipmi_address': '10.0.220.172',
+                    'ipmi_gateway': '10.0.220.161',
+                    'host_profile': 'dp-r720'
+                },
+                'cab2r73c15': {
+                    'ipmi_address': '10.0.220.173',
+                    'ipmi_gateway': '10.0.220.161',
+                    'host_profile': 'dp-r720'
+                },
+                'cab2r73c16': {
+                    'ipmi_address': '10.0.220.174',
+                    'ipmi_gateway': '10.0.220.161',
+                    'host_profile': 'cp-r720'
+                },
+                'cab2r73c17': {
+                    'ipmi_address': '10.0.220.175',
+                    'ipmi_gateway': '10.0.220.161',
+                    'host_profile': 'cp-r720'
+                },
+            },
+            [
+                'cab2r72c12',
+                'cab2r72c13',
+                'cab2r72c14',
+                'cab2r72c15',
+                'cab2r72c16',
+                'cab2r72c17',
+                'cab2r73c12',
+                'cab2r73c13',
+                'cab2r73c14',
+                'cab2r73c15',
+                'cab2r73c16',
+                'cab2r73c17',
+            ]
+        ],
+        'network_data': {
+            'private': {
+                'iSCSI/Storage': {
+                    'vlan': 'vlan 23',
+                    'subnet': ['30.31.1.0/25'],
+                    'is_common': True
+                },
+                'PXE': {
+                    'vlan': 'vlan 21',
+                    'subnet': [
+                        '30.30.4.0/25', '30.30.4.128/25', '30.30.5.0/25',
+                        '30.30.5.128/25'
+                    ],
+                    'is_common': True
+                },
+                'Calico BGP peering addresses': {
+                    'vlan': 'vlan 22',
+                    'subnet': ['30.29.1.0/25'],
+                    'is_common': True
+                },
+                'Overlay': {
+                    'vlan': 'vlan 24',
+                    'subnet': ['30.19.0.0/25'],
+                    'is_common': True
+                }
+            },
+            'public': {
+                'oam': {
+                    'subnet': ['10.0.220.0/26'],
+                    'vlan': 'VLAN-21'
+                },
+                'ingress': '10.0.220.72/29',
+                'oob': {
+                    'subnet': [
+                        '10.0.220.128/27', '10.0.220.160/27',
+                        '10.0.220.192/27', '10.0.220.224/27'
+                    ]
+                }
+            }
+        },
+        'site_info': {
+            'location': {
+                'corridor': 'Corridor 1',
+                'name': 'SampleSiteName',
+                'state': 'New Jersey',
+                'country': 'SampleCountry',
+                'physical_location': 'XXXXXX21'
+            },
+            'dns': [
+                '40.40.40.40', '(ntp1.example.com)', '41.41.41.41',
+                '(ntp2.example.com)'
+            ],
+            'ntp': ['150.234.210.5', '(ns1.example.com)'],
+            'domain': 'dmy00.example.com',
+            'ldap': {
+                'subdomain': 'testitservices',
+                'common_name': 'AA-AAA-dmy00',
+                'url': 'url: ldap://ldap.example.com'
+            }
+        }
+    }

tests/shared/excel_spec.yaml

@@ -1,5 +1,4 @@
-# Copyright 2018 The Openstack-Helm Authors.
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
-# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,6 +18,8 @@
 specs:
   # Design Spec file name: SiteDesignSpec_v0.1.xlsx
   xl_spec:
+    header_row: 3
+    ipmi_address_header: "IPMI Address"
     ipmi_sheet_name: 'Site-Information'
     start_row: 4
     end_row: 15

tests/shared/invalid_excel_spec.yaml

@@ -0,0 +1,64 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Important: Please modify the dictionary with appropriate
+# design spec file.
+---
+specs:
+  # Design Spec file name: SiteDesignSpec_v0.1.xlsx
+  xl_spec:
+    header_row: 3
+    ipmi_address_header: "IPMI Address"
+    ipmi_sheet_name: 'Sheet-DNE'
+    start_row: 4
+    end_row: 15
+    hostname_col: 2
+    ipmi_address_col: 2
+    host_profile_col: 5
+    ipmi_gateway_col: 7
+    private_ip_sheet: 'Site-Information'
+    net_type_col: 1
+    vlan_col: 2
+    vlan_start_row: 19
+    vlan_end_row: 30
+    net_start_row: 33
+    net_end_row: 40
+    net_col: 2
+    net_vlan_col: 1
+    public_ip_sheet: 'Site-Information'
+    oam_vlan_col: 1
+    oam_ip_row: 43
+    oam_ip_col: 2
+    oob_net_row: 48
+    oob_net_start_col: 2
+    oob_net_end_col: 5
+    ingress_ip_row: 45
+    dns_ntp_ldap_sheet: 'Site-Information'
+    login_domain_row: 52
+    ldap_col: 2
+    global_group: 53
+    ldap_search_url_row: 54
+    ntp_row: 55
+    ntp_col: 2
+    dns_row: 56
+    dns_col: 2
+    domain_row: 51
+    domain_col: 2
+    location_sheet: 'Site-Information'
+    column: 2
+    corridor_row: 59
+    site_name_row: 58
+    state_name_row: 60
+    country_name_row: 61
+    clli_name_row: 62

tests/shared/site_config.yaml

@@ -1,3 +1,16 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 ##############################################
 # Site Specific Spyglass XLS Plugin Settings #
 ##############################################

tests/unit/test_cli.py

@@ -18,8 +18,8 @@ from unittest import mock
 from click.testing import CliRunner
 from spyglass.site_processors.site_processor import SiteProcessor
 
-from spyglass_plugin_xls.cli import generate_intermediary, \
+from spyglass_plugin_xls.cli import generate_intermediary
-    generate_manifests_and_intermediary
+from spyglass_plugin_xls.cli import generate_manifests_and_intermediary
 
 FIXTURE_DIR = os.path.join(
     os.path.dirname(os.path.dirname(__file__)), 'shared')

tests/unit/test_excel.py

@@ -0,0 +1,355 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from copy import copy
+import os
+import unittest
+from unittest import mock
+
+import pytest
+from spyglass.data_extractor import models
+
+from spyglass_plugin_xls.check_exceptions import ExcelFileNotSpecified
+from spyglass_plugin_xls.check_exceptions import ExcelSpecNotSpecified
+from spyglass_plugin_xls.excel import ExcelPlugin
+from spyglass_plugin_xls.excel_parser import ExcelParser
+
+FIXTURE_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(__file__)), 'shared')
+
+EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'excel_spec.yaml')
+
+INVALID_EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'invalid_excel_spec.yaml')
+
+EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx')
+
+SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml')
+
+
+@pytest.mark.usefixtures('site_data')
+class TestExcelPlugin(unittest.TestCase):
+    """Tests for ExcelPlugin"""
+
+    def test___init__(self):
+        region = 'test_region'
+        result = ExcelPlugin(region)
+        self.assertEqual(region, result.region)
+        self.assertEqual('excel', result.source_type)
+        self.assertEqual('spyglass-plugin-xls', result.source_name)
+        self.assertEqual(None, result.excel_path)
+        self.assertEqual(None, result.excel_spec)
+        self.assertEqual(None, result.parsed_xl_data)
+
+    @mock.patch('spyglass_plugin_xls.excel_parser.ExcelParser', autospec=True)
+    def test_set_config_opts(self, excel_parser):
+        region = 'test_region'
+        result = ExcelPlugin(region)
+        config = {'excel_spec': EXCEL_SPEC_PATH, 'excel_path': EXCEL_FILE_PATH}
+        result.excel_spec = EXCEL_SPEC_PATH
+        result.excel_path = EXCEL_FILE_PATH
+        result.set_config_opts(config)
+        self.assertEqual(config['excel_path'], result.excel_path)
+        self.assertEqual(config['excel_spec'], result.excel_spec)
+        self.assertIsInstance(result.excel_obj, ExcelParser)
+
+    def test_get_plugin_conf(self):
+        expected_result = {
+            'excel_path': 'ExcelFile.xlsx',
+            'excel_spec': 'ExcelSpec.yaml'
+        }
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        result = obj.get_plugin_conf(
+            excel_file='ExcelFile.xlsx', excel_spec='ExcelSpec.yaml')
+        self.assertDictEqual(expected_result, result)
+
+    def test_get_plugin_conf_no_excel_file(self):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        with self.assertRaises(ExcelFileNotSpecified):
+            obj.get_plugin_conf(excel_spec='ExcelSpec.yaml')
+
+    def test_get_plugin_conf_no_excel_spec(self):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        with self.assertRaises(ExcelSpecNotSpecified):
+            obj.get_plugin_conf(excel_file='ExcelFile.xlsx')
+
+    def test_get_racks(self):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_racks(region)
+        self.assertEqual(2, len(result))
+        for rack in result:
+            self.assertIsInstance(rack, models.Rack)
+            self.assertEqual(6, len(rack.hosts))
+            for host in rack.hosts:
+                self.assertIn(host.name, self.site_data['ipmi_data'][0])
+                self.assertEqual(
+                    self.site_data['ipmi_data'][0][host.name]['host_profile'],
+                    host.host_profile)
+
+    def test_get_hosts(self):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_hosts(region)
+        self.assertEqual(12, len(result))
+        for host in result:
+            self.assertIn(host.name, self.site_data['ipmi_data'][0])
+            self.assertEqual(
+                self.site_data['ipmi_data'][0][host.name]['host_profile'],
+                host.host_profile)
+
+    def test_get_hosts_using_rack(self):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_hosts(region, 'rack73')
+        self.assertEqual(6, len(result))
+        for host in result:
+            self.assertIn(host.name, self.site_data['ipmi_data'][0])
+            self.assertEqual('rack73', host.rack_name)
+            self.assertNotIn('r72', host.name)
+            self.assertEqual(
+                self.site_data['ipmi_data'][0][host.name]['host_profile'],
+                host.host_profile)
+
+    def test_get_networks(self):
+        expected_network_types = {
+            'oob': {
+                'type': 'public',
+                'name': 'oob'
+            },
+            'oam': {
+                'type': 'public',
+                'name': 'oam'
+            },
+            'calico': {
+                'type': 'private',
+                'name': 'Calico BGP peering addresses'
+            },
+            'ingress': {
+                'type': 'public',
+                'name': 'ingress'
+            },
+            'overlay': {
+                'type': 'private',
+                'name': 'Overlay'
+            },
+            'pxe': {
+                'type': 'private',
+                'name': 'PXE'
+            },
+            'storage': {
+                'type': 'private',
+                'name': 'iSCSI/Storage'
+            }
+        }
+        network_data = self.site_data['network_data']
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_networks(region)
+        self.assertEqual(7, len(result))
+        for vlan_data in result:
+            self.assertIn(vlan_data.name, expected_network_types)
+            data = expected_network_types[vlan_data.name]
+            if vlan_data.name != 'ingress':
+                self.assertEqual(
+                    network_data[data['type']][data['name']]['subnet'],
+                    vlan_data.subnet)
+            else:
+                self.assertEqual(
+                    network_data[data['type']][data['name']],
+                    vlan_data.subnet[0])
+
+    def test_get_ips(self):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        host_name = 'cab2r72c15'
+        result = obj.get_ips(region, host_name)
+        self.assertIsInstance(result, models.IPList)
+        self.assertEqual(
+            self.site_data['ipmi_data'][0][host_name]['ipmi_address'],
+            result.oob)
+
+    def test_get_ldap_information(self):
+        expected_ldap_data = copy(self.site_data['site_info']['ldap'])
+        expected_ldap_data['domain'] = 'example'
+        expected_ldap_data['url'] = expected_ldap_data['url'].split(' ')[1]
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_ldap_information(region)
+        self.assertDictEqual(expected_ldap_data, result)
+
+    def test_get_ntp_servers(self):
+        expected_ntp_servers = self.site_data['site_info']['ntp'][:1]
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_ntp_servers(region)
+        self.assertIsInstance(result, models.ServerList)
+        self.assertEqual(expected_ntp_servers, result.servers)
+
+    def test_get_dns_servers(self):
+        expected_dns_servers = [
+            self.site_data['site_info']['dns'][0],
+            self.site_data['site_info']['dns'][2]
+        ]
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_dns_servers(region)
+        self.assertIsInstance(result, models.ServerList)
+        self.assertEqual(expected_dns_servers, result.servers)
+
+    def test_get_domain_name(self):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_domain_name(region)
+        self.assertEqual(self.site_data['site_info']['domain'], result)
+
+    def test_get_location_information(self):
+        expected_location_data = copy(self.site_data['site_info']['location'])
+        expected_location_data['corridor'] = 'c1'
+        expected_location_data[
+            'physical_location_id'] = expected_location_data.pop(
+                'physical_location')
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_location_information(region)
+        self.assertDictEqual(expected_location_data, result)
+
+    def test_get_site_info(self):
+        expected_ntp_servers = self.site_data['site_info']['ntp'][:1]
+        expected_dns_servers = [
+            self.site_data['site_info']['dns'][0],
+            self.site_data['site_info']['dns'][2]
+        ]
+
+        expected_location_data = copy(self.site_data['site_info']['location'])
+        expected_location_data['corridor'] = 'c1'
+        expected_location_data[
+            'physical_location_id'] = expected_location_data.pop(
+                'physical_location')
+
+        expected_ldap_data = copy(self.site_data['site_info']['ldap'])
+        expected_ldap_data['domain'] = 'example'
+        expected_ldap_data['url'] = expected_ldap_data['url'].split(' ')[1]
+
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj.get_site_info(region)
+        self.assertIsInstance(result, models.SiteInfo)
+        self.assertEqual(region, result.region_name)
+        self.assertEqual(expected_dns_servers, result.dns.servers)
+        self.assertEqual(expected_ntp_servers, result.ntp.servers)
+        self.assertDictEqual(expected_ldap_data, result.ldap)
+        self.assertEqual(expected_location_data['corridor'], result.corridor)
+        self.assertEqual(expected_location_data['state'], result.state)
+        self.assertEqual(expected_location_data['country'], result.country)
+        self.assertEqual(
+            expected_location_data['physical_location_id'],
+            result.physical_location_id)
+        self.assertEqual(expected_location_data['name'], result.name)
+
+    @mock.patch('spyglass_plugin_xls.excel_parser.ExcelParser')
+    def test__get_excel_obj(self, excel_parser):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.excel_spec = EXCEL_SPEC_PATH
+        obj.excel_path = EXCEL_FILE_PATH
+        obj._get_excel_obj()
+        self.assertIsInstance(obj.excel_obj, ExcelParser)
+
+    def test__extract_raw_data_from_excel(self):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.excel_obj = mock.MagicMock(spec=ExcelParser)
+        obj.excel_obj.get_data.return_value = 'success'
+        obj._extract_raw_data_from_excel()
+        obj.excel_obj.get_data.assert_called_once()
+        self.assertEqual('success', obj.parsed_xl_data)
+
+    def test__get_network_name_from_vlan_name(self):
+        result = ExcelPlugin._get_network_name_from_vlan_name('ksn')
+        self.assertEqual('calico', result)
+        result = ExcelPlugin._get_network_name_from_vlan_name('calico')
+        self.assertEqual('calico', result)
+        result = ExcelPlugin._get_network_name_from_vlan_name('storage')
+        self.assertEqual('storage', result)
+        result = ExcelPlugin._get_network_name_from_vlan_name('oam')
+        self.assertEqual('oam', result)
+        result = ExcelPlugin._get_network_name_from_vlan_name('server')
+        self.assertEqual('oam', result)
+        result = ExcelPlugin._get_network_name_from_vlan_name('ovs')
+        self.assertEqual('overlay', result)
+        result = ExcelPlugin._get_network_name_from_vlan_name('overlay')
+        self.assertEqual('overlay', result)
+        result = ExcelPlugin._get_network_name_from_vlan_name('oob')
+        self.assertEqual('oob', result)
+        result = ExcelPlugin._get_network_name_from_vlan_name('pxe')
+        self.assertEqual('pxe', result)
+
+    def test__get_network_name_from_vlan_name_dne(self):
+        result = ExcelPlugin._get_network_name_from_vlan_name('dne')
+        self.assertEqual('', result)
+
+    def test__get_formatted_server_list(self):
+        test_list = [
+            '124.1.23.54', '(example.com)', '192.168.1.0',
+            '(anotherexample.com)'
+        ]
+        expected_list = ['124.1.23.54', '192.168.1.0']
+        result = ExcelPlugin._get_formatted_server_list(test_list)
+        self.assertIsInstance(result, models.ServerList)
+        self.assertEqual(expected_list, result.servers)
+
+    def test__get_rack(self):
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        result = obj._get_rack('cab2r72c15')
+        self.assertEqual('r72', result)
+
+    def test__get_rackwise_hosts(self):
+        expected_data = {
+            'rack72': [
+                'cab2r72c12', 'cab2r72c13', 'cab2r72c14', 'cab2r72c15',
+                'cab2r72c16', 'cab2r72c17'
+            ],
+            'rack73': [
+                'cab2r73c12', 'cab2r73c13', 'cab2r73c14', 'cab2r73c15',
+                'cab2r73c16', 'cab2r73c17'
+            ]
+        }
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj._get_rackwise_hosts()
+        self.assertDictEqual(expected_data, result)
+
+    def test__get_rack_data(self):
+        expected_data = {'r72': 'rack72', 'r73': 'rack73'}
+        region = 'test_region'
+        obj = ExcelPlugin(region)
+        obj.parsed_xl_data = self.site_data
+        result = obj._get_rack_data()
+        self.assertDictEqual(expected_data, result)

tests/unit/test_excel_parser.py

@@ -0,0 +1,168 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+from openpyxl import Workbook
+from openpyxl.worksheet.worksheet import Worksheet
+import pytest
+from spyglass.data_extractor.custom_exceptions import NoSpecMatched
+import yaml
+
+from spyglass_plugin_xls.excel_parser import ExcelParser
+
+FIXTURE_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(__file__)), 'shared')
+
+EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'excel_spec.yaml')
+
+INVALID_EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'invalid_excel_spec.yaml')
+
+EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx')
+
+SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml')
+
+
+@pytest.mark.usefixtures('site_data')
+class TestExcelParser(unittest.TestCase):
+    """Tests for ExcelParser"""
+
+    def test___init__(self):
+        with open(EXCEL_SPEC_PATH, 'r') as f:
+            loaded_spec = yaml.safe_load(f)
+        result = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        self.assertEqual(EXCEL_FILE_PATH, result.file_name)
+        self.assertDictEqual(loaded_spec, result.excel_specs)
+        self.assertIsInstance(result.wb_combined, Workbook)
+        self.assertEqual('xl_spec', result.spec)
+
+    def test_sanitize(self):
+        test_string = 'Hello THIS is A TeSt'
+        expected_output = 'hellothisisatest'
+        result = ExcelParser.sanitize(test_string)
+        self.assertEqual(expected_output, result)
+
+    def test_compare(self):
+        test_string1 = 'These strings are equal.'
+        test_string2 = 'These strIngs are Equal .'
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.compare(test_string1, test_string2)
+        self.assertTrue(result)
+
+    def test_compare_false(self):
+        test_string1 = 'These strings are not equal.'
+        test_string2 = 'These strIngs are Equal.'
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.compare(test_string1, test_string2)
+        self.assertFalse(result)
+
+    def test_validate_sheet(self):
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.validate_sheet('xl_spec', 'Site-Information')
+        self.assertTrue(result)
+
+    def test_validate_sheet_invalid(self):
+        obj = ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
+        result = obj.validate_sheet('xl_spec', 'Site-Information')
+        self.assertFalse(result)
+
+    def test_find_correct_spec(self):
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.find_correct_spec()
+        self.assertEqual('xl_spec', result)
+
+    def test_find_correct_spec_no_spec_matched(self):
+        obj = ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
+        with self.assertRaises(NoSpecMatched):
+            obj.find_correct_spec()
+
+    def test__get_workbook(self):
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj._get_workbook()
+        self.assertIsInstance(result, Worksheet)
+
+    def test_get_ipmi_data(self):
+        expected_hosts = self.site_data['ipmi_data'][1]
+        expected_ipmi_data = self.site_data['ipmi_data'][0]
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_ipmi_data()
+        self.assertDictEqual(result[0], expected_ipmi_data)
+        self.assertEqual(result[1], expected_hosts)
+
+    def test_get_private_vlan_data(self):
+        expected_vlan_data = {
+            'vlan 23': 'iSCSI/Storage',
+            'vlan 21': 'PXE',
+            'vlan 22': 'Calico BGP peering addresses',
+            'vlan 24': 'Overlay',
+            'n/a': 'CNI Pod addresses'
+        }
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_private_vlan_data(obj._get_workbook())
+        self.assertDictEqual(expected_vlan_data, result)
+
+    def test_get_private_network_data(self):
+        expected_network_data = self.site_data['network_data']['private']
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_private_network_data()
+        self.assertDictEqual(expected_network_data, result)
+
+    def test_get_public_network_data(self):
+        expected_network_data = self.site_data['network_data']['public']
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_public_network_data()
+        self.assertEqual(expected_network_data, result)
+
+    def test_get_site_info(self):
+        expected_site_info = self.site_data['site_info']
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_site_info()
+        self.assertDictEqual(expected_site_info, result)
+
+    def test_get_location_data(self):
+        expected_location_data = self.site_data['site_info']['location']
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_location_data()
+        self.assertEqual(expected_location_data, result)
+
+    def test_validate_sheet_names_with_spec(self):
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        self.assertIsNone(obj.validate_sheet_names_with_spec())
+
+    def test_validate_sheet_names_with_spec_invalid(self):
+        obj = ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
+        with self.assertRaises(RuntimeError):
+            obj.validate_sheet_names_with_spec()
+
+    def test_get_data(self):
+        expected_data = self.site_data
+        obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_data()
+        self.assertDictEqual(expected_data, result)
+
+    def test_load_excel_data(self):
+        result = ExcelParser.load_excel_data(EXCEL_FILE_PATH)
+        self.assertIsInstance(result, Workbook)
+
+    def test_get_xl_obj_and_sheetname(self):
+        result = ExcelParser.get_xl_obj_and_sheetname('Site-Information')
+        self.assertEqual([None, 'Site-Information'], result)
+
+    def test_get_xl_obj_and_sheetname_file_specified(self):
+        sheet = EXCEL_FILE_PATH + ':Site-Information'
+        result = ExcelParser.get_xl_obj_and_sheetname(sheet)
+        self.assertIsInstance(result, list)
+        self.assertIsInstance(result[0], Workbook)
+        self.assertEqual(result[1], 'Site-Information')

tox.ini

@@ -33,7 +33,7 @@ deps =
 commands =
   bash -c "{toxinidir}/tools/gate/whitespace-linter.sh"
   yapf -dr {toxinidir}/spyglass_plugin_xls {toxinidir}/setup.py {toxinidir}/tests
-  flake8 {toxinidir}/spyglass_plugin_xls
+  flake8 {toxinidir}/spyglass_plugin_xls {toxinidir}/tests
   bandit -r spyglass_plugin_xls -n 5
   safety check -r requirements.txt --bare
 whitelist_externals =
@@ -72,6 +72,6 @@ deps =
 commands =
   bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=spyglass_plugin_xls \
       --cov-report html:cover --cov-report xml:cover/coverage.xml \
-      --cov-report term --cov-fail-under 20 tests/'
+      --cov-report term --cov-fail-under 92 tests/'
 whitelist_externals =
   bash