From a4c09b699fbe9f802673e2c891f50b0c6529fd65 Mon Sep 17 00:00:00 2001 From: Ryan Schroder Date: Fri, 22 Nov 2019 15:34:35 -0600 Subject: [PATCH] Allow excel plugin to take csv files as an input Checks if file is csv and coverts csv data to a new workbook Change-Id: I69f53268870f9293172ac3c50ab9553db0061352 --- doc/source/index.rst | 38 +++-- .../examples/Site-Information.csv | 63 +++++++++ spyglass_plugin_xls/excel_parser.py | 29 +++- tests/shared/Site-Information.csv | 63 +++++++++ tests/unit/test_excel_parser.py | 133 ++++++++++++++++++ 5 files changed, 310 insertions(+), 16 deletions(-) create mode 100644 spyglass_plugin_xls/examples/Site-Information.csv create mode 100644 tests/shared/Site-Information.csv diff --git a/doc/source/index.rst b/doc/source/index.rst index c7e7824..2aa9c51 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -25,9 +25,9 @@ What is the Spyglass XLS Plugin? The Spyglass XLS plugin is used to generate airship-seaworthy site manifest files from an excel based engineering spec. The plugin is configured with an -Excel sheet and its corresponding excel specification as inputs. Spyglass uses -this plugin to construct an intermediary yaml which is processed further using -J2 templates to generate site manifests. +Excel sheet or a CSV file and its corresponding excel specification as inputs. +Spyglass uses this plugin to construct an intermediary yaml which is processed +further using Jinja2 templates to generate site manifests. Excel Specification ------------------- @@ -38,7 +38,8 @@ need to be filled by the Deployment Engineer. Below is the definition for each key in the Excel spec * ipmi_sheet_name - name of the sheet from where IPMI and host profile - information is to be read + information is to be read (File name, excluding the file extension, when + using a CSV file) * start_row - row number from where the IPMI and host profile information starts * end_row - row number from where the IPMI and host profile information ends @@ -88,21 +89,34 @@ Below is the definition for each key in the Excel spec * country_name_row - row number which has the country name * clli_name_row - row number which has CLLI information -Example: Tugboat Plugin Usage ------------------------------ +Example: Spyglass XLS Plugin Usage +---------------------------------- -1. Required Input (Refer to 'spyglass/examples' folder to get these inputs) +1. Required Input (Refer to 'spyglass_plugin_xls/examples' folder to get these +inputs) a) Excel File: SiteDesignSpec_v0.1.xlsx b) Excel Spec: excel_spec_upstream.yaml c) Site Config: site_config.yaml - d) Template_dir: '../examples/templates' + d) Template_dir: 'spyglass/examples/templates' e) Site name: airship-seaworthy -2. Spyglass CLI Command: +2. Spyglass CLI Command using an Excel File: .. code-block:: bash - spyglass m -i -p tugboat -x SiteDesignSpec_v0.1.xlsx \ - -e excel_spec_upstream.yaml -c site_config.yaml \ - -s airship-seaworthy -t \ No newline at end of file + spyglass excel documents -i \ + -x ../spyglass-plugin-xls/spyglass_plugin_xls/examples/SiteDesignSpec_v0.1.xlsx \ + -e ../spyglass-plugin-xls/spyglass_plugin_xls/examples/excel_spec.yaml \ + -c ../spyglass-plugin-xls/spyglass_plugin_xls/examples/site_config.yaml \ + -s airship-seaworthy -t spyglass/examples/templates/ + +3. Spyglass CLI Command using a CSV File: + +.. code-block:: bash + + spyglass excel documents -i \ + -x ../spyglass-plugin-xls/spyglass_plugin_xls/examples/Site-Information.csv \ + -e ../spyglass-plugin-xls/spyglass_plugin_xls/examples/excel_spec.yaml \ + -c ../spyglass-plugin-xls/spyglass_plugin_xls/examples/site_config.yaml \ + -s airship-seaworthy -t spyglass/examples/templates/ \ No newline at end of file diff --git a/spyglass_plugin_xls/examples/Site-Information.csv b/spyglass_plugin_xls/examples/Site-Information.csv new file mode 100644 index 0000000..41accd6 --- /dev/null +++ b/spyglass_plugin_xls/examples/Site-Information.csv @@ -0,0 +1,63 @@ +Site-Information,,,, +IPMI Host Details,,,, +,Server Name,IPMI Address,IPMI Gateway,Host Profile +,cab2r72c12,10.0.220.138,10.0.220.129,dp-r720 +,cab2r72c13,10.0.220.139,10.0.220.129,dp-r720 +,cab2r72c14,10.0.220.140,10.0.220.129,dp-r720 +,cab2r72c15,10.0.220.141,10.0.220.129,dp-r720 +,cab2r72c16,10.0.220.142,10.0.220.129,cp-r720 +,cab2r72c17,10.0.220.143,10.0.220.129,cp-r720 +,cab2r73c12,10.0.220.170,10.0.220.161,dp-r720 +,cab2r73c13,10.0.220.171,10.0.220.161,dp-r720 +,cab2r73c14,10.0.220.172,10.0.220.161,dp-r720 +,cab2r73c15,10.0.220.173,10.0.220.161,dp-r720 +,cab2r73c16,10.0.220.174,10.0.220.161,cp-r720 +,cab2r73c17,10.0.220.175,10.0.220.161,cp-r720 +,,,, +Network Details,,,, +Usage,vLAN,,, +iSCSI/Storage,vLAN 23,,, +PXE,vLAN 21,,, +Calico BGP peering addresses,vLAN 22,,, +Overlay,vLAN 24,,, +CNI Pod addresses,N/A,,, +,,,, +,,,, +,,,, +,,,, +,,,, +,,,, +,,,, +,,,, +VLAN,POD,,, +vLAN 23,30.31.1.0/25,,, +vLAN 21,30.30.4.0/25,,, +vLAN 21,30.30.4.128/25,,, +vLAN 21,30.30.5.0/25,,, +vLAN 21,30.30.5.128/25,,, +,,,, +vLAN 22,30.29.1.0/25,,, +vLAN 24,30.19.0.0/25,,, +,,,, +VLAN,Rack1,Rack2,Rack3,Rack4 +VLAN-21,10.0.220.0/26,,, +VLAN-49,10.0.220.64/29,,, +Calico Ingress,10.0.220.72/29,,, +Loopback 3 ,10.0.220.80/27,,, +Spare,10.0.220.112-10.0.220.127,,, +VLAN 1023,10.0.220.128/27,10.0.220.160/27,10.0.220.192/27,10.0.220.224/27 +SiteDetails,,,, +,,,, +Domain,dmy00.example.com,,, +Subdomain,testitservices,,, +Group,AA-AAA-dmy00,,, +ldap_url,url: ldap://ldap.example.com,,, +NTP,150.234.210.5 (ns1.example.com),,, +DNS,"40.40.40.40 (ntp1.example.com), +41.41.41.41 (ntp2.example.com)",,, +,,,, +Sitename,SampleSiteName,,, +Corridor,Corridor 1,,, +State,New Jersey,,, +Country,SampleCountry,,, +Clli,XXXXXX21,,, diff --git a/spyglass_plugin_xls/excel_parser.py b/spyglass_plugin_xls/excel_parser.py index 045af24..5bbe932 100644 --- a/spyglass_plugin_xls/excel_parser.py +++ b/spyglass_plugin_xls/excel_parser.py @@ -13,7 +13,9 @@ # limitations under the License. from copy import deepcopy +import csv import logging +import os import pprint import re @@ -316,10 +318,19 @@ class ExcelParser(object): @staticmethod def load_excel_data(filename): - """Combines multiple excel file to a single design spec""" + """Combines multiple excel or csv files to a single design spec""" design_spec = Workbook() - loaded_workbook = load_workbook(filename, data_only=True) + if os.path.splitext(filename)[1] == '.csv': + loaded_workbook = Workbook() + ws = loaded_workbook.active + with open(filename) as f: + reader = csv.reader(f, delimiter=',') + for row in reader: + ws.append(row) + ws.title = os.path.splitext(os.path.basename(filename))[0] + else: + loaded_workbook = load_workbook(filename, data_only=True) for names in loaded_workbook.sheetnames: design_spec_worksheet = design_spec.create_sheet(names) loaded_workbook_ws = loaded_workbook[names] @@ -334,11 +345,21 @@ class ExcelParser(object): 'MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs' """ - - if re.search(".xlsx", sheetname) or re.search(".xls", sheetname): + file_type = os.path.splitext(sheetname.split(':')[0])[1] + if file_type == '.xlsx' or file_type == '.xls': # Extract file name source_xl_file = sheetname.split(":")[0] wb = load_workbook(source_xl_file, data_only=True) return [wb, sheetname.split(":")[1]] + elif file_type == ".csv": + source_csv_file = sheetname.split(":")[0] + wb = Workbook() + ws = wb.active + with open(source_csv_file) as f: + reader = csv.reader(f, delimiter=',') + for row in reader: + ws.append(row) + ws.title = os.path.splitext(os.path.basename(source_csv_file))[0] + return [wb, sheetname.split(":")[1]] else: return [None, sheetname] diff --git a/tests/shared/Site-Information.csv b/tests/shared/Site-Information.csv new file mode 100644 index 0000000..41accd6 --- /dev/null +++ b/tests/shared/Site-Information.csv @@ -0,0 +1,63 @@ +Site-Information,,,, +IPMI Host Details,,,, +,Server Name,IPMI Address,IPMI Gateway,Host Profile +,cab2r72c12,10.0.220.138,10.0.220.129,dp-r720 +,cab2r72c13,10.0.220.139,10.0.220.129,dp-r720 +,cab2r72c14,10.0.220.140,10.0.220.129,dp-r720 +,cab2r72c15,10.0.220.141,10.0.220.129,dp-r720 +,cab2r72c16,10.0.220.142,10.0.220.129,cp-r720 +,cab2r72c17,10.0.220.143,10.0.220.129,cp-r720 +,cab2r73c12,10.0.220.170,10.0.220.161,dp-r720 +,cab2r73c13,10.0.220.171,10.0.220.161,dp-r720 +,cab2r73c14,10.0.220.172,10.0.220.161,dp-r720 +,cab2r73c15,10.0.220.173,10.0.220.161,dp-r720 +,cab2r73c16,10.0.220.174,10.0.220.161,cp-r720 +,cab2r73c17,10.0.220.175,10.0.220.161,cp-r720 +,,,, +Network Details,,,, +Usage,vLAN,,, +iSCSI/Storage,vLAN 23,,, +PXE,vLAN 21,,, +Calico BGP peering addresses,vLAN 22,,, +Overlay,vLAN 24,,, +CNI Pod addresses,N/A,,, +,,,, +,,,, +,,,, +,,,, +,,,, +,,,, +,,,, +,,,, +VLAN,POD,,, +vLAN 23,30.31.1.0/25,,, +vLAN 21,30.30.4.0/25,,, +vLAN 21,30.30.4.128/25,,, +vLAN 21,30.30.5.0/25,,, +vLAN 21,30.30.5.128/25,,, +,,,, +vLAN 22,30.29.1.0/25,,, +vLAN 24,30.19.0.0/25,,, +,,,, +VLAN,Rack1,Rack2,Rack3,Rack4 +VLAN-21,10.0.220.0/26,,, +VLAN-49,10.0.220.64/29,,, +Calico Ingress,10.0.220.72/29,,, +Loopback 3 ,10.0.220.80/27,,, +Spare,10.0.220.112-10.0.220.127,,, +VLAN 1023,10.0.220.128/27,10.0.220.160/27,10.0.220.192/27,10.0.220.224/27 +SiteDetails,,,, +,,,, +Domain,dmy00.example.com,,, +Subdomain,testitservices,,, +Group,AA-AAA-dmy00,,, +ldap_url,url: ldap://ldap.example.com,,, +NTP,150.234.210.5 (ns1.example.com),,, +DNS,"40.40.40.40 (ntp1.example.com), +41.41.41.41 (ntp2.example.com)",,, +,,,, +Sitename,SampleSiteName,,, +Corridor,Corridor 1,,, +State,New Jersey,,, +Country,SampleCountry,,, +Clli,XXXXXX21,,, diff --git a/tests/unit/test_excel_parser.py b/tests/unit/test_excel_parser.py index 5088c23..51e68ca 100644 --- a/tests/unit/test_excel_parser.py +++ b/tests/unit/test_excel_parser.py @@ -32,6 +32,8 @@ INVALID_EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'invalid_excel_spec.yaml') EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx') +CSV_FILE_PATH = os.path.join(FIXTURE_DIR, "Site-Information.csv") + SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml') @@ -55,6 +57,18 @@ class TestExcelParser(unittest.TestCase): self.assertIsInstance(result.wb_combined, Workbook) self.assertEqual('xl_spec', result.spec) + def test___init__csv(self): + with open(EXCEL_SPEC_PATH, 'r') as f: + loaded_spec = yaml.safe_load(f) + result = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) + self.assertEqual(CSV_FILE_PATH, result.file_name) + self.assertDictEqual(loaded_spec, result.excel_specs) + self.assertDictEqual( + loaded_spec['specs'][self.SPEC], result.loaded_spec) + self.assertDictEqual(self.raw_excel_data, result.loaded_data) + self.assertIsInstance(result.wb_combined, Workbook) + self.assertEqual('xl_spec', result.spec) + def test_sanitize(self): test_string = 'Hello THIS is A TeSt' expected_output = 'hellothisisatest' @@ -68,6 +82,13 @@ class TestExcelParser(unittest.TestCase): result = obj.compare(test_string1, test_string2) self.assertTrue(result) + def test_compare_csv(self): + test_string1 = 'These strings are equal.' + test_string2 = 'These strIngs are Equal .' + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.compare(test_string1, test_string2) + self.assertTrue(result) + def test_compare_false(self): test_string1 = 'These strings are not equal.' test_string2 = 'These strIngs are Equal.' @@ -75,11 +96,23 @@ class TestExcelParser(unittest.TestCase): result = obj.compare(test_string1, test_string2) self.assertFalse(result) + def test_compare_false_csv(self): + test_string1 = 'These strings are not equal.' + test_string2 = 'These strIngs are Equal.' + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.compare(test_string1, test_string2) + self.assertFalse(result) + def test__get_workbook(self): obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) result = obj._get_workbook('Site-Information') self.assertIsInstance(result, Worksheet) + def test__get_workbook_csv(self): + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj._get_workbook('Site-Information') + self.assertIsInstance(result, Worksheet) + def test__check_sanitize_settings(self): test_data_sanitize_only = {'sanitize': False} sanitize, no_sanitize_keys = ExcelParser._check_sanitize_settings( @@ -102,30 +135,61 @@ class TestExcelParser(unittest.TestCase): obj.loaded_spec['public']['data']['oam'], 'Site-Information') self.assertDictEqual(expected_data, result) + def test_extract_data_points_csv(self): + expected_data = self.raw_excel_data['public']['oam'] + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) + result = obj.extract_data_points( + obj.loaded_spec['public']['data']['oam'], 'Site-Information') + self.assertDictEqual(expected_data, result) + def test_extract_data_points_unsanitized(self): expected_data = self.raw_excel_data['location'] obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) result = obj.extract_data_points(obj.loaded_spec['location']) self.assertDictEqual(expected_data, result) + def test_extract_data_points_unsanitized_csv(self): + expected_data = self.raw_excel_data['location'] + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) + result = obj.extract_data_points(obj.loaded_spec['location']) + self.assertDictEqual(expected_data, result) + def test_extract_data_series(self): expected_data = self.raw_excel_data['ipmi'] obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) result = obj.extract_data_series(obj.loaded_spec['ipmi']) self.assertEqual(expected_data, result) + def test_extract_data_series_csv(self): + expected_data = self.raw_excel_data['ipmi'] + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) + result = obj.extract_data_series(obj.loaded_spec['ipmi']) + self.assertEqual(expected_data, result) + def test_extract_data_series_no_sanitize(self): expected_data = self.raw_excel_data['private_vlan'] obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) result = obj.extract_data_series(obj.loaded_spec['private_vlan']) self.assertEqual(expected_data, result) + def test_extract_data_series_no_sanitize_csv(self): + expected_data = self.raw_excel_data['private_vlan'] + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) + result = obj.extract_data_series(obj.loaded_spec['private_vlan']) + self.assertEqual(expected_data, result) + def test_extract_data_using_spec(self): expected_data = self.raw_excel_data obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) result = obj.extract_data_using_spec(obj.loaded_spec) self.assertDictEqual(expected_data, result) + def test_extract_data_using_spec_csv(self): + expected_data = self.raw_excel_data + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC) + result = obj.extract_data_using_spec(obj.loaded_spec) + self.assertDictEqual(expected_data, result) + def test_get_ipmi_data(self): expected_hosts = self.site_data['ipmi_data'][1] expected_ipmi_data = self.site_data['ipmi_data'][0] @@ -134,6 +198,14 @@ class TestExcelParser(unittest.TestCase): self.assertDictEqual(result[0], expected_ipmi_data) self.assertEqual(result[1], expected_hosts) + def test_get_ipmi_data_csv(self): + expected_hosts = self.site_data['ipmi_data'][1] + expected_ipmi_data = self.site_data['ipmi_data'][0] + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_ipmi_data() + self.assertDictEqual(result[0], expected_ipmi_data) + self.assertEqual(result[1], expected_hosts) + def test_get_private_vlan_data(self): expected_vlan_data = { 'vlan23': 'iSCSI/Storage', @@ -146,48 +218,102 @@ class TestExcelParser(unittest.TestCase): result = obj.get_private_vlan_data() self.assertDictEqual(expected_vlan_data, result) + def test_get_private_vlan_data_csv(self): + expected_vlan_data = { + 'vlan23': 'iSCSI/Storage', + 'vlan21': 'PXE', + 'vlan22': 'Calico BGP peering addresses', + 'vlan24': 'Overlay', + 'na': 'CNI Pod addresses' + } + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_private_vlan_data() + self.assertDictEqual(expected_vlan_data, result) + def test_get_private_network_data(self): expected_network_data = self.site_data['network_data']['private'] obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) result = obj.get_private_network_data() self.assertDictEqual(expected_network_data, result) + def test_get_private_network_data_csv(self): + expected_network_data = self.site_data['network_data']['private'] + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_private_network_data() + self.assertDictEqual(expected_network_data, result) + def test_get_public_network_data(self): expected_network_data = self.site_data['network_data']['public'] obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) result = obj.get_public_network_data() self.assertEqual(expected_network_data, result) + def test_get_public_network_data_csv(self): + expected_network_data = self.site_data['network_data']['public'] + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_public_network_data() + self.assertEqual(expected_network_data, result) + def test_get_site_info(self): expected_site_info = self.site_data['site_info'] obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) result = obj.get_site_info() self.assertDictEqual(expected_site_info, result) + def test_get_site_info_csv(self): + expected_site_info = self.site_data['site_info'] + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_site_info() + self.assertDictEqual(expected_site_info, result) + def test_get_location_data(self): expected_location_data = self.site_data['site_info']['location'] obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) result = obj.get_location_data() self.assertEqual(expected_location_data, result) + def test_get_location_data_csv(self): + expected_location_data = self.site_data['site_info']['location'] + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_location_data() + self.assertEqual(expected_location_data, result) + def test_validate_sheet_names_with_spec(self): obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) self.assertIsNone(obj.validate_sheet_names_with_spec()) + def test_validate_sheet_names_with_spec_csv(self): + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + self.assertIsNone(obj.validate_sheet_names_with_spec()) + def test_validate_sheet_names_with_spec_invalid(self): with self.assertRaises(ExcelSheetNotFound): ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH) + def test_validate_sheet_names_with_spec_invalid_csv(self): + with self.assertRaises(ExcelSheetNotFound): + ExcelParser(CSV_FILE_PATH, INVALID_EXCEL_SPEC_PATH) + def test_get_data(self): expected_data = self.site_data obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH) result = obj.get_data() self.assertDictEqual(expected_data, result) + def test_get_data_csv(self): + expected_data = self.site_data + obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH) + result = obj.get_data() + self.assertDictEqual(expected_data, result) + def test_load_excel_data(self): result = ExcelParser.load_excel_data(EXCEL_FILE_PATH) self.assertIsInstance(result, Workbook) + def test_load_csv_data(self): + result = ExcelParser.load_excel_data(CSV_FILE_PATH) + self.assertIsInstance(result, Workbook) + def test_get_xl_obj_and_sheetname(self): result = ExcelParser.get_xl_obj_and_sheetname('Site-Information') self.assertEqual([None, 'Site-Information'], result) @@ -198,3 +324,10 @@ class TestExcelParser(unittest.TestCase): self.assertIsInstance(result, list) self.assertIsInstance(result[0], Workbook) self.assertEqual(result[1], 'Site-Information') + + def test_get_xl_obj_and_sheetname_file_specified_csv(self): + sheet = CSV_FILE_PATH + ':Site-Information' + result = ExcelParser.get_xl_obj_and_sheetname(sheet) + self.assertIsInstance(result, list) + self.assertIsInstance(result[0], Workbook) + self.assertEqual(result[1], 'Site-Information')