airship/spyglass-plugin-xls
Code Issues Proposed changes

Testing for plugin

Browse Source 
Adds tests for the Excel plugin's parser and extractor.

Enables pep8 and fmt checks on the tests directory.

Increases plugin test coverage to 94%, sets new minimum to 92%.

Fixes DNS and NTP server extraction with regex.

Updates file licenses.

Change-Id: I35ee97574e6d63b7a82cfa94caf79db5db9755e7
This commit is contained in:
Ian H Pittwood 2019-06-26 16:41:42 -05:00
parent d3212bd367
commit e84b80c32d
14 changed files with 859 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
spyglass_plugin_xls/check_exceptions.py
View File

@ -1,4 +1,4 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -36,3 +36,17 @@ class NoSpecMatched(BaseError):
print(
"No spec matched. Following are the available specs:\n".format(
self.specs))
class ExcelFileNotSpecified(BaseError):
@staticmethod
def display_error():
print("Engineering excel file not specified")
class ExcelSpecNotSpecified(BaseError):
@staticmethod
def display_error():
print("Engineering excel spec not specified")

2
spyglass_plugin_xls/cli.py
View File

@ -1,4 +1,4 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

3
spyglass_plugin_xls/examples/excel_spec.yaml
View File

@ -1,5 +1,4 @@
# Copyright 2018 The Openstack-Helm Authors.
# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

13
spyglass_plugin_xls/examples/site_config.yaml
View File

@ -1,3 +1,16 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##############################################
# Site Specific Spyglass XLS Plugin Settings #
##############################################

25
spyglass_plugin_xls/excel.py
View File

@ -1,4 +1,4 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,6 +18,9 @@ import pprint
import re
from spyglass.data_extractor.base import BaseDataSourcePlugin
from spyglass.data_extractor import models
from spyglass_plugin_xls.check_exceptions import ExcelFileNotSpecified
from spyglass_plugin_xls.check_exceptions import ExcelSpecNotSpecified
from spyglass_plugin_xls.excel_parser import ExcelParser
LOG = logging.getLogger(__name__)
@ -66,13 +69,11 @@ class ExcelPlugin(BaseDataSourcePlugin):
and excel specs are not specified. The below code has been
written as an additional safeguard.
"""
if not kwargs["excel_file"]:
LOG.error("Engineering excel file not specified: Spyglass exited!")
exit()
if 'excel_file' not in kwargs:
raise ExcelFileNotSpecified()
excel_file_info = kwargs["excel_file"]
if not kwargs["excel_spec"]:
LOG.error("Engineering spec file not specified: Spyglass exited!")
exit()
if 'excel_spec' not in kwargs:
raise ExcelSpecNotSpecified()
excel_spec_info = kwargs["excel_spec"]
plugin_conf = {
"excel_path": excel_file_info,
@ -161,7 +162,7 @@ class ExcelPlugin(BaseDataSourcePlugin):
vlan_list.append(models.VLANNetworkData(**tmp_vlan))
return vlan_list
def get_ips(self, region, host=None):
def get_ips(self, region, host):
"""Return list of IPs on the host
:param string region: Region name
@ -255,7 +256,7 @@ class ExcelPlugin(BaseDataSourcePlugin):
data_dict['dns'] = self.get_dns_servers(region)
data_dict['ntp'] = self.get_ntp_servers(region)
data_dict['ldap'] = self.get_ldap_information(region)
return models.SiteInfo(region, **data_dict)
return models.SiteInfo(region_name=region, **data_dict)
def _get_excel_obj(self):
"""Creation of an ExcelParser object to store site information.
@ -271,7 +272,8 @@ class ExcelPlugin(BaseDataSourcePlugin):
"""Extracts raw information from excel file based on excel spec"""
self.parsed_xl_data = self.excel_obj.get_data()
def _get_network_name_from_vlan_name(self, vlan_name):
@staticmethod
def _get_network_name_from_vlan_name(vlan_name):
"""Network names are ksn, oam, oob, overlay, storage, pxe
@ -317,7 +319,8 @@ class ExcelPlugin(BaseDataSourcePlugin):
"Unable to recognize VLAN name extracted from Plugin data source")
return ""
def _get_formatted_server_list(self, server_list):
@staticmethod
def _get_formatted_server_list(server_list):
"""Format dns and ntp server list as comma separated string"""
# dns/ntp server info from excel is of the format

98
spyglass_plugin_xls/excel_parser.py
View File

@ -1,4 +1,4 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -30,16 +30,20 @@ LOG = logging.getLogger(__name__)
class ExcelParser(object):
"""Parse data from excel into a dict"""
def __init__(self, file_name, excel_specs):
def __init__(self, file_name: str, excel_specs: str):
"""Initializes an ExcelParser to extract data from the Excel workbook
:param file_name: path to the Excel workbook
:param excel_specs: path to the Excel workbook spec
"""
self.file_name = file_name
with open(excel_specs, "r") as f:
spec_raw_data = f.read()
self.excel_specs = yaml.safe_load(spec_raw_data)
# A combined design spec, returns a workbook object after combining
# all the inputs excel specs
combined_design_spec = self.combine_excel_design_specs(file_name)
combined_design_spec = self.load_excel_data(file_name)
self.wb_combined = combined_design_spec
self.filenames = file_name
self.spec = "xl_spec"
@staticmethod
@ -75,19 +79,22 @@ class ExcelParser(object):
return spec
raise NoSpecMatched(self.excel_specs)
def get_ipmi_data(self):
"""Read IPMI data from the sheet"""
ipmi_data = {}
hosts = []
def _get_workbook(self):
provided_sheetname = self.excel_specs["specs"][
self.spec]["ipmi_sheet_name"]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
return workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
return self.wb_combined[provided_sheetname]
def get_ipmi_data(self):
"""Read IPMI data from the sheet"""
ipmi_data = {}
hosts = []
ws = self._get_workbook()
row = self.excel_specs["specs"][self.spec]["start_row"]
end_row = self.excel_specs["specs"][self.spec]["end_row"]
hostname_col = self.excel_specs["specs"][self.spec]["hostname_col"]
@ -119,7 +126,7 @@ class ExcelParser(object):
self.spec, row, host_profile_col))
except RuntimeError as rerror:
LOG.critical(rerror)
sys.exit("Tugboat exited!!")
sys.exit("Spyglass exited")
ipmi_data[hostname] = {
"ipmi_address": ipmi_address,
"ipmi_gateway": ipmi_gateway,
@ -157,14 +164,7 @@ class ExcelParser(object):
def get_private_network_data(self):
"""Read network data from the private ip sheet"""
provided_sheetname = self.excel_specs["specs"][
self.spec]["private_ip_sheet"]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
ws = self._get_workbook()
vlan_data = self.get_private_vlan_data(ws)
network_data = {}
row = self.excel_specs["specs"][self.spec]["net_start_row"]
@ -208,14 +208,7 @@ class ExcelParser(object):
"""Read public network data from public ip data"""
network_data = {}
provided_sheetname = self.excel_specs["specs"][
self.spec]["public_ip_sheet"]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
ws = self._get_workbook()
oam_row = self.excel_specs["specs"][self.spec]["oam_ip_row"]
oam_col = self.excel_specs["specs"][self.spec]["oam_ip_col"]
oam_vlan_col = self.excel_specs["specs"][self.spec]["oam_vlan_col"]
@ -250,13 +243,8 @@ class ExcelParser(object):
site_info = {}
provided_sheetname = self.excel_specs["specs"][
self.spec]["dns_ntp_ldap_sheet"]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
self.spec]["ipmi_sheet_name"]
ws = self._get_workbook()
dns_row = self.excel_specs["specs"][self.spec]["dns_row"]
dns_col = self.excel_specs["specs"][self.spec]["dns_col"]
ntp_row = self.excel_specs["specs"][self.spec]["ntp_row"]
@ -282,17 +270,8 @@ class ExcelParser(object):
except RuntimeError as rerror:
LOG.critical(rerror)
sys.exit("Tugboat exited!!")
dns_servers = dns_servers.replace("\n", " ")
ntp_servers = ntp_servers.replace("\n", " ")
if "," in dns_servers:
dns_servers = dns_servers.split(",")
else:
dns_servers = dns_servers.split()
if "," in ntp_servers:
ntp_servers = ntp_servers.split(",")
else:
ntp_servers = ntp_servers.split()
dns_servers = list(filter(None, re.split(" |,|\n", dns_servers)))
ntp_servers = list(filter(None, re.split(" |,|\n", ntp_servers)))
site_info = {
"location": self.get_location_data(),
"dns": dns_servers,
@ -316,14 +295,7 @@ class ExcelParser(object):
def get_location_data(self):
"""Read location data from the site and zone sheet"""
provided_sheetname = self.excel_specs["specs"][
self.spec]["location_sheet"]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
ws = self._get_workbook()
corridor_row = self.excel_specs["specs"][self.spec]["corridor_row"]
column = self.excel_specs["specs"][self.spec]["column"]
site_name_row = self.excel_specs["specs"][self.spec]["site_name_row"]
@ -356,7 +328,6 @@ class ExcelParser(object):
sheet_name_list.append(dns_ntp_ldap_sheet_name)
location_sheet_name = spec_item["location_sheet"]
sheet_name_list.append(location_sheet_name)
try:
for sheetname in sheet_name_list:
workbook_object, extracted_sheetname = (
self.get_xl_obj_and_sheetname(sheetname))
@ -369,9 +340,6 @@ class ExcelParser(object):
if sheetname not in wb.sheetnames:
raise RuntimeError(
"SheetName '{}' not found ".format(sheetname))
except RuntimeError as rerror:
LOG.critical(rerror)
sys.exit("Tugboat exited!!")
LOG.info("Sheet names in excel spec validated")
@ -398,29 +366,29 @@ class ExcelParser(object):
)
return data
def combine_excel_design_specs(self, filenames):
@staticmethod
def load_excel_data(filename):
"""Combines multiple excel file to a single design spec"""
design_spec = Workbook()
for exel_file in filenames:
loaded_workbook = load_workbook(exel_file, data_only=True)
loaded_workbook = load_workbook(filename, data_only=True)
for names in loaded_workbook.sheetnames:
design_spec_worksheet = design_spec.create_sheet(names)
loaded_workbook_ws = loaded_workbook[names]
for row in loaded_workbook_ws:
for cell in row:
design_spec_worksheet[
cell.coordinate].value = cell.value
design_spec_worksheet[cell.coordinate].value = cell.value
return design_spec
def get_xl_obj_and_sheetname(self, sheetname):
@staticmethod
def get_xl_obj_and_sheetname(sheetname):
"""The logic confirms if the sheetname is specified for example as:
'MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs'
"""
if re.search(".xlsx", sheetname) or re.search(".xls", sheetname):
""" Extract file name """
# Extract file name
source_xl_file = sheetname.split(":")[0]
wb = load_workbook(source_xl_file, data_only=True)
return [wb, sheetname.split(":")[1]]

159
tests/conftest.py Normal file
View File

@ -0,0 +1,159 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
@pytest.fixture(scope='class')
def site_data(request):
request.cls.site_data = {
'ipmi_data': [
{
'cab2r72c12': {
'ipmi_address': '10.0.220.138',
'ipmi_gateway': '10.0.220.129',
'host_profile': 'dp-r720'
},
'cab2r72c13': {
'ipmi_address': '10.0.220.139',
'ipmi_gateway': '10.0.220.129',
'host_profile': 'dp-r720'
},
'cab2r72c14': {
'ipmi_address': '10.0.220.140',
'ipmi_gateway': '10.0.220.129',
'host_profile': 'dp-r720'
},
'cab2r72c15': {
'ipmi_address': '10.0.220.141',
'ipmi_gateway': '10.0.220.129',
'host_profile': 'dp-r720'
},
'cab2r72c16': {
'ipmi_address': '10.0.220.142',
'ipmi_gateway': '10.0.220.129',
'host_profile': 'cp-r720'
},
'cab2r72c17': {
'ipmi_address': '10.0.220.143',
'ipmi_gateway': '10.0.220.129',
'host_profile': 'cp-r720'
},
'cab2r73c12': {
'ipmi_address': '10.0.220.170',
'ipmi_gateway': '10.0.220.161',
'host_profile': 'dp-r720'
},
'cab2r73c13': {
'ipmi_address': '10.0.220.171',
'ipmi_gateway': '10.0.220.161',
'host_profile': 'dp-r720'
},
'cab2r73c14': {
'ipmi_address': '10.0.220.172',
'ipmi_gateway': '10.0.220.161',
'host_profile': 'dp-r720'
},
'cab2r73c15': {
'ipmi_address': '10.0.220.173',
'ipmi_gateway': '10.0.220.161',
'host_profile': 'dp-r720'
},
'cab2r73c16': {
'ipmi_address': '10.0.220.174',
'ipmi_gateway': '10.0.220.161',
'host_profile': 'cp-r720'
},
'cab2r73c17': {
'ipmi_address': '10.0.220.175',
'ipmi_gateway': '10.0.220.161',
'host_profile': 'cp-r720'
},
},
[
'cab2r72c12',
'cab2r72c13',
'cab2r72c14',
'cab2r72c15',
'cab2r72c16',
'cab2r72c17',
'cab2r73c12',
'cab2r73c13',
'cab2r73c14',
'cab2r73c15',
'cab2r73c16',
'cab2r73c17',
]
],
'network_data': {
'private': {
'iSCSI/Storage': {
'vlan': 'vlan 23',
'subnet': ['30.31.1.0/25'],
'is_common': True
},
'PXE': {
'vlan': 'vlan 21',
'subnet': [
'30.30.4.0/25', '30.30.4.128/25', '30.30.5.0/25',
'30.30.5.128/25'
],
'is_common': True
},
'Calico BGP peering addresses': {
'vlan': 'vlan 22',
'subnet': ['30.29.1.0/25'],
'is_common': True
},
'Overlay': {
'vlan': 'vlan 24',
'subnet': ['30.19.0.0/25'],
'is_common': True
}
},
'public': {
'oam': {
'subnet': ['10.0.220.0/26'],
'vlan': 'VLAN-21'
},
'ingress': '10.0.220.72/29',
'oob': {
'subnet': [
'10.0.220.128/27', '10.0.220.160/27',
'10.0.220.192/27', '10.0.220.224/27'
]
}
}
},
'site_info': {
'location': {
'corridor': 'Corridor 1',
'name': 'SampleSiteName',
'state': 'New Jersey',
'country': 'SampleCountry',
'physical_location': 'XXXXXX21'
},
'dns': [
'40.40.40.40', '(ntp1.example.com)', '41.41.41.41',
'(ntp2.example.com)'
],
'ntp': ['150.234.210.5', '(ns1.example.com)'],
'domain': 'dmy00.example.com',
'ldap': {
'subdomain': 'testitservices',
'common_name': 'AA-AAA-dmy00',
'url': 'url: ldap://ldap.example.com'
}
}
}

5
tests/shared/excel_spec.yaml
View File

@ -1,5 +1,4 @@
# Copyright 2018 The Openstack-Helm Authors.
# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -19,6 +18,8 @@
specs:
# Design Spec file name: SiteDesignSpec_v0.1.xlsx
xl_spec:
header_row: 3
ipmi_address_header: "IPMI Address"
ipmi_sheet_name: 'Site-Information'
start_row: 4
end_row: 15

64
tests/shared/invalid_excel_spec.yaml Normal file
View File

@ -0,0 +1,64 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Important: Please modify the dictionary with appropriate
# design spec file.
---
specs:
# Design Spec file name: SiteDesignSpec_v0.1.xlsx
xl_spec:
header_row: 3
ipmi_address_header: "IPMI Address"
ipmi_sheet_name: 'Sheet-DNE'
start_row: 4
end_row: 15
hostname_col: 2
ipmi_address_col: 2
host_profile_col: 5
ipmi_gateway_col: 7
private_ip_sheet: 'Site-Information'
net_type_col: 1
vlan_col: 2
vlan_start_row: 19
vlan_end_row: 30
net_start_row: 33
net_end_row: 40
net_col: 2
net_vlan_col: 1
public_ip_sheet: 'Site-Information'
oam_vlan_col: 1
oam_ip_row: 43
oam_ip_col: 2
oob_net_row: 48
oob_net_start_col: 2
oob_net_end_col: 5
ingress_ip_row: 45
dns_ntp_ldap_sheet: 'Site-Information'
login_domain_row: 52
ldap_col: 2
global_group: 53
ldap_search_url_row: 54
ntp_row: 55
ntp_col: 2
dns_row: 56
dns_col: 2
domain_row: 51
domain_col: 2
location_sheet: 'Site-Information'
column: 2
corridor_row: 59
site_name_row: 58
state_name_row: 60
country_name_row: 61
clli_name_row: 62

13
tests/shared/site_config.yaml
View File

@ -1,3 +1,16 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##############################################
# Site Specific Spyglass XLS Plugin Settings #
##############################################

4
tests/unit/test_cli.py
View File

@ -18,8 +18,8 @@ from unittest import mock
from click.testing import CliRunner
from spyglass.site_processors.site_processor import SiteProcessor
from spyglass_plugin_xls.cli import generate_intermediary, \
generate_manifests_and_intermediary
from spyglass_plugin_xls.cli import generate_intermediary
from spyglass_plugin_xls.cli import generate_manifests_and_intermediary
FIXTURE_DIR = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'shared')

355
tests/unit/test_excel.py Normal file
View File

@ -0,0 +1,355 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from copy import copy
import os
import unittest
from unittest import mock
import pytest
from spyglass.data_extractor import models
from spyglass_plugin_xls.check_exceptions import ExcelFileNotSpecified
from spyglass_plugin_xls.check_exceptions import ExcelSpecNotSpecified
from spyglass_plugin_xls.excel import ExcelPlugin
from spyglass_plugin_xls.excel_parser import ExcelParser
FIXTURE_DIR = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'shared')
EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'excel_spec.yaml')
INVALID_EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'invalid_excel_spec.yaml')
EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx')
SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml')
@pytest.mark.usefixtures('site_data')
class TestExcelPlugin(unittest.TestCase):
"""Tests for ExcelPlugin"""
def test___init__(self):
region = 'test_region'
result = ExcelPlugin(region)
self.assertEqual(region, result.region)
self.assertEqual('excel', result.source_type)
self.assertEqual('spyglass-plugin-xls', result.source_name)
self.assertEqual(None, result.excel_path)
self.assertEqual(None, result.excel_spec)
self.assertEqual(None, result.parsed_xl_data)
@mock.patch('spyglass_plugin_xls.excel_parser.ExcelParser', autospec=True)
def test_set_config_opts(self, excel_parser):
region = 'test_region'
result = ExcelPlugin(region)
config = {'excel_spec': EXCEL_SPEC_PATH, 'excel_path': EXCEL_FILE_PATH}
result.excel_spec = EXCEL_SPEC_PATH
result.excel_path = EXCEL_FILE_PATH
result.set_config_opts(config)
self.assertEqual(config['excel_path'], result.excel_path)
self.assertEqual(config['excel_spec'], result.excel_spec)
self.assertIsInstance(result.excel_obj, ExcelParser)
def test_get_plugin_conf(self):
expected_result = {
'excel_path': 'ExcelFile.xlsx',
'excel_spec': 'ExcelSpec.yaml'
}
region = 'test_region'
obj = ExcelPlugin(region)
result = obj.get_plugin_conf(
excel_file='ExcelFile.xlsx', excel_spec='ExcelSpec.yaml')
self.assertDictEqual(expected_result, result)
def test_get_plugin_conf_no_excel_file(self):
region = 'test_region'
obj = ExcelPlugin(region)
with self.assertRaises(ExcelFileNotSpecified):
obj.get_plugin_conf(excel_spec='ExcelSpec.yaml')
def test_get_plugin_conf_no_excel_spec(self):
region = 'test_region'
obj = ExcelPlugin(region)
with self.assertRaises(ExcelSpecNotSpecified):
obj.get_plugin_conf(excel_file='ExcelFile.xlsx')
def test_get_racks(self):
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_racks(region)
self.assertEqual(2, len(result))
for rack in result:
self.assertIsInstance(rack, models.Rack)
self.assertEqual(6, len(rack.hosts))
for host in rack.hosts:
self.assertIn(host.name, self.site_data['ipmi_data'][0])
self.assertEqual(
self.site_data['ipmi_data'][0][host.name]['host_profile'],
host.host_profile)
def test_get_hosts(self):
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_hosts(region)
self.assertEqual(12, len(result))
for host in result:
self.assertIn(host.name, self.site_data['ipmi_data'][0])
self.assertEqual(
self.site_data['ipmi_data'][0][host.name]['host_profile'],
host.host_profile)
def test_get_hosts_using_rack(self):
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_hosts(region, 'rack73')
self.assertEqual(6, len(result))
for host in result:
self.assertIn(host.name, self.site_data['ipmi_data'][0])
self.assertEqual('rack73', host.rack_name)
self.assertNotIn('r72', host.name)
self.assertEqual(
self.site_data['ipmi_data'][0][host.name]['host_profile'],
host.host_profile)
def test_get_networks(self):
expected_network_types = {
'oob': {
'type': 'public',
'name': 'oob'
},
'oam': {
'type': 'public',
'name': 'oam'
},
'calico': {
'type': 'private',
'name': 'Calico BGP peering addresses'
},
'ingress': {
'type': 'public',
'name': 'ingress'
},
'overlay': {
'type': 'private',
'name': 'Overlay'
},
'pxe': {
'type': 'private',
'name': 'PXE'
},
'storage': {
'type': 'private',
'name': 'iSCSI/Storage'
}
}
network_data = self.site_data['network_data']
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_networks(region)
self.assertEqual(7, len(result))
for vlan_data in result:
self.assertIn(vlan_data.name, expected_network_types)
data = expected_network_types[vlan_data.name]
if vlan_data.name != 'ingress':
self.assertEqual(
network_data[data['type']][data['name']]['subnet'],
vlan_data.subnet)
else:
self.assertEqual(
network_data[data['type']][data['name']],
vlan_data.subnet[0])
def test_get_ips(self):
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
host_name = 'cab2r72c15'
result = obj.get_ips(region, host_name)
self.assertIsInstance(result, models.IPList)
self.assertEqual(
self.site_data['ipmi_data'][0][host_name]['ipmi_address'],
result.oob)
def test_get_ldap_information(self):
expected_ldap_data = copy(self.site_data['site_info']['ldap'])
expected_ldap_data['domain'] = 'example'
expected_ldap_data['url'] = expected_ldap_data['url'].split(' ')[1]
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_ldap_information(region)
self.assertDictEqual(expected_ldap_data, result)
def test_get_ntp_servers(self):
expected_ntp_servers = self.site_data['site_info']['ntp'][:1]
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_ntp_servers(region)
self.assertIsInstance(result, models.ServerList)
self.assertEqual(expected_ntp_servers, result.servers)
def test_get_dns_servers(self):
expected_dns_servers = [
self.site_data['site_info']['dns'][0],
self.site_data['site_info']['dns'][2]
]
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_dns_servers(region)
self.assertIsInstance(result, models.ServerList)
self.assertEqual(expected_dns_servers, result.servers)
def test_get_domain_name(self):
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_domain_name(region)
self.assertEqual(self.site_data['site_info']['domain'], result)
def test_get_location_information(self):
expected_location_data = copy(self.site_data['site_info']['location'])
expected_location_data['corridor'] = 'c1'
expected_location_data[
'physical_location_id'] = expected_location_data.pop(
'physical_location')
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_location_information(region)
self.assertDictEqual(expected_location_data, result)
def test_get_site_info(self):
expected_ntp_servers = self.site_data['site_info']['ntp'][:1]
expected_dns_servers = [
self.site_data['site_info']['dns'][0],
self.site_data['site_info']['dns'][2]
]
expected_location_data = copy(self.site_data['site_info']['location'])
expected_location_data['corridor'] = 'c1'
expected_location_data[
'physical_location_id'] = expected_location_data.pop(
'physical_location')
expected_ldap_data = copy(self.site_data['site_info']['ldap'])
expected_ldap_data['domain'] = 'example'
expected_ldap_data['url'] = expected_ldap_data['url'].split(' ')[1]
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj.get_site_info(region)
self.assertIsInstance(result, models.SiteInfo)
self.assertEqual(region, result.region_name)
self.assertEqual(expected_dns_servers, result.dns.servers)
self.assertEqual(expected_ntp_servers, result.ntp.servers)
self.assertDictEqual(expected_ldap_data, result.ldap)
self.assertEqual(expected_location_data['corridor'], result.corridor)
self.assertEqual(expected_location_data['state'], result.state)
self.assertEqual(expected_location_data['country'], result.country)
self.assertEqual(
expected_location_data['physical_location_id'],
result.physical_location_id)
self.assertEqual(expected_location_data['name'], result.name)
@mock.patch('spyglass_plugin_xls.excel_parser.ExcelParser')
def test__get_excel_obj(self, excel_parser):
region = 'test_region'
obj = ExcelPlugin(region)
obj.excel_spec = EXCEL_SPEC_PATH
obj.excel_path = EXCEL_FILE_PATH
obj._get_excel_obj()
self.assertIsInstance(obj.excel_obj, ExcelParser)
def test__extract_raw_data_from_excel(self):
region = 'test_region'
obj = ExcelPlugin(region)
obj.excel_obj = mock.MagicMock(spec=ExcelParser)
obj.excel_obj.get_data.return_value = 'success'
obj._extract_raw_data_from_excel()
obj.excel_obj.get_data.assert_called_once()
self.assertEqual('success', obj.parsed_xl_data)
def test__get_network_name_from_vlan_name(self):
result = ExcelPlugin._get_network_name_from_vlan_name('ksn')
self.assertEqual('calico', result)
result = ExcelPlugin._get_network_name_from_vlan_name('calico')
self.assertEqual('calico', result)
result = ExcelPlugin._get_network_name_from_vlan_name('storage')
self.assertEqual('storage', result)
result = ExcelPlugin._get_network_name_from_vlan_name('oam')
self.assertEqual('oam', result)
result = ExcelPlugin._get_network_name_from_vlan_name('server')
self.assertEqual('oam', result)
result = ExcelPlugin._get_network_name_from_vlan_name('ovs')
self.assertEqual('overlay', result)
result = ExcelPlugin._get_network_name_from_vlan_name('overlay')
self.assertEqual('overlay', result)
result = ExcelPlugin._get_network_name_from_vlan_name('oob')
self.assertEqual('oob', result)
result = ExcelPlugin._get_network_name_from_vlan_name('pxe')
self.assertEqual('pxe', result)
def test__get_network_name_from_vlan_name_dne(self):
result = ExcelPlugin._get_network_name_from_vlan_name('dne')
self.assertEqual('', result)
def test__get_formatted_server_list(self):
test_list = [
'124.1.23.54', '(example.com)', '192.168.1.0',
'(anotherexample.com)'
]
expected_list = ['124.1.23.54', '192.168.1.0']
result = ExcelPlugin._get_formatted_server_list(test_list)
self.assertIsInstance(result, models.ServerList)
self.assertEqual(expected_list, result.servers)
def test__get_rack(self):
region = 'test_region'
obj = ExcelPlugin(region)
result = obj._get_rack('cab2r72c15')
self.assertEqual('r72', result)
def test__get_rackwise_hosts(self):
expected_data = {
'rack72': [
'cab2r72c12', 'cab2r72c13', 'cab2r72c14', 'cab2r72c15',
'cab2r72c16', 'cab2r72c17'
],
'rack73': [
'cab2r73c12', 'cab2r73c13', 'cab2r73c14', 'cab2r73c15',
'cab2r73c16', 'cab2r73c17'
]
}
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj._get_rackwise_hosts()
self.assertDictEqual(expected_data, result)
def test__get_rack_data(self):
expected_data = {'r72': 'rack72', 'r73': 'rack73'}
region = 'test_region'
obj = ExcelPlugin(region)
obj.parsed_xl_data = self.site_data
result = obj._get_rack_data()
self.assertDictEqual(expected_data, result)

168
tests/unit/test_excel_parser.py Normal file
View File

@ -0,0 +1,168 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import unittest
from openpyxl import Workbook
from openpyxl.worksheet.worksheet import Worksheet
import pytest
from spyglass.data_extractor.custom_exceptions import NoSpecMatched
import yaml
from spyglass_plugin_xls.excel_parser import ExcelParser
FIXTURE_DIR = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'shared')
EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'excel_spec.yaml')
INVALID_EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'invalid_excel_spec.yaml')
EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx')
SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml')
@pytest.mark.usefixtures('site_data')
class TestExcelParser(unittest.TestCase):
"""Tests for ExcelParser"""
def test___init__(self):
with open(EXCEL_SPEC_PATH, 'r') as f:
loaded_spec = yaml.safe_load(f)
result = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
self.assertEqual(EXCEL_FILE_PATH, result.file_name)
self.assertDictEqual(loaded_spec, result.excel_specs)
self.assertIsInstance(result.wb_combined, Workbook)
self.assertEqual('xl_spec', result.spec)
def test_sanitize(self):
test_string = 'Hello THIS is A TeSt'
expected_output = 'hellothisisatest'
result = ExcelParser.sanitize(test_string)
self.assertEqual(expected_output, result)
def test_compare(self):
test_string1 = 'These strings are equal.'
test_string2 = 'These strIngs are Equal .'
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.compare(test_string1, test_string2)
self.assertTrue(result)
def test_compare_false(self):
test_string1 = 'These strings are not equal.'
test_string2 = 'These strIngs are Equal.'
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.compare(test_string1, test_string2)
self.assertFalse(result)
def test_validate_sheet(self):
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.validate_sheet('xl_spec', 'Site-Information')
self.assertTrue(result)
def test_validate_sheet_invalid(self):
obj = ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
result = obj.validate_sheet('xl_spec', 'Site-Information')
self.assertFalse(result)
def test_find_correct_spec(self):
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.find_correct_spec()
self.assertEqual('xl_spec', result)
def test_find_correct_spec_no_spec_matched(self):
obj = ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
with self.assertRaises(NoSpecMatched):
obj.find_correct_spec()
def test__get_workbook(self):
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj._get_workbook()
self.assertIsInstance(result, Worksheet)
def test_get_ipmi_data(self):
expected_hosts = self.site_data['ipmi_data'][1]
expected_ipmi_data = self.site_data['ipmi_data'][0]
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_ipmi_data()
self.assertDictEqual(result[0], expected_ipmi_data)
self.assertEqual(result[1], expected_hosts)
def test_get_private_vlan_data(self):
expected_vlan_data = {
'vlan 23': 'iSCSI/Storage',
'vlan 21': 'PXE',
'vlan 22': 'Calico BGP peering addresses',
'vlan 24': 'Overlay',
'n/a': 'CNI Pod addresses'
}
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_private_vlan_data(obj._get_workbook())
self.assertDictEqual(expected_vlan_data, result)
def test_get_private_network_data(self):
expected_network_data = self.site_data['network_data']['private']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_private_network_data()
self.assertDictEqual(expected_network_data, result)
def test_get_public_network_data(self):
expected_network_data = self.site_data['network_data']['public']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_public_network_data()
self.assertEqual(expected_network_data, result)
def test_get_site_info(self):
expected_site_info = self.site_data['site_info']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_site_info()
self.assertDictEqual(expected_site_info, result)
def test_get_location_data(self):
expected_location_data = self.site_data['site_info']['location']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_location_data()
self.assertEqual(expected_location_data, result)
def test_validate_sheet_names_with_spec(self):
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
self.assertIsNone(obj.validate_sheet_names_with_spec())
def test_validate_sheet_names_with_spec_invalid(self):
obj = ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
with self.assertRaises(RuntimeError):
obj.validate_sheet_names_with_spec()
def test_get_data(self):
expected_data = self.site_data
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_data()
self.assertDictEqual(expected_data, result)
def test_load_excel_data(self):
result = ExcelParser.load_excel_data(EXCEL_FILE_PATH)
self.assertIsInstance(result, Workbook)
def test_get_xl_obj_and_sheetname(self):
result = ExcelParser.get_xl_obj_and_sheetname('Site-Information')
self.assertEqual([None, 'Site-Information'], result)
def test_get_xl_obj_and_sheetname_file_specified(self):
sheet = EXCEL_FILE_PATH + ':Site-Information'
result = ExcelParser.get_xl_obj_and_sheetname(sheet)
self.assertIsInstance(result, list)
self.assertIsInstance(result[0], Workbook)
self.assertEqual(result[1], 'Site-Information')

4
tox.ini
View File

@ -33,7 +33,7 @@ deps =
commands =
bash -c "{toxinidir}/tools/gate/whitespace-linter.sh"
yapf -dr {toxinidir}/spyglass_plugin_xls {toxinidir}/setup.py {toxinidir}/tests
flake8 {toxinidir}/spyglass_plugin_xls
flake8 {toxinidir}/spyglass_plugin_xls {toxinidir}/tests
bandit -r spyglass_plugin_xls -n 5
safety check -r requirements.txt --bare
whitelist_externals =
@ -72,6 +72,6 @@ deps =
commands =
bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=spyglass_plugin_xls \
--cov-report html:cover --cov-report xml:cover/coverage.xml \
--cov-report term --cov-fail-under 20 tests/'
--cov-report term --cov-fail-under 92 tests/'
whitelist_externals =
bash