Allow excel plugin to take csv files as an input

Checks if file is csv and coverts csv data to a new workbook

Change-Id: I69f53268870f9293172ac3c50ab9553db0061352

doc/source/index.rst

@@ -25,9 +25,9 @@ What is the Spyglass XLS Plugin?
 
 The Spyglass XLS plugin is used to generate airship-seaworthy site manifest
 files from an excel based engineering spec. The plugin is configured with an
-Excel sheet and its corresponding excel specification as inputs. Spyglass uses
+Excel sheet or a CSV file and its corresponding excel specification as inputs.
-this plugin to construct an intermediary yaml which is processed further using
+Spyglass uses this plugin to construct an intermediary yaml which is processed
-J2 templates to generate site manifests.
+further using Jinja2 templates to generate site manifests.
 
 Excel Specification
 -------------------
@@ -38,7 +38,8 @@ need to be filled by the Deployment Engineer.
 Below is the definition for each key in the Excel spec
 
 * ipmi_sheet_name - name of the sheet from where IPMI and host profile
-  information is to be read
+  information is to be read (File name, excluding the file extension, when
+  using a CSV file)
 * start_row - row number from where the IPMI and host profile information
   starts
 * end_row - row number from where the IPMI and host profile information ends
@@ -88,21 +89,34 @@ Below is the definition for each key in the Excel spec
 * country_name_row - row number which has the country name
 * clli_name_row - row number which has CLLI information
 
-Example: Tugboat Plugin Usage
+Example: Spyglass XLS Plugin Usage
------------------------------
+----------------------------------
 
-1. Required Input (Refer to 'spyglass/examples' folder to get these inputs)
+1. Required Input (Refer to 'spyglass_plugin_xls/examples' folder to get these
+inputs)
 
     a) Excel File: SiteDesignSpec_v0.1.xlsx
     b) Excel Spec: excel_spec_upstream.yaml
     c) Site Config: site_config.yaml
-    d) Template_dir: '../examples/templates'
+    d) Template_dir: 'spyglass/examples/templates'
     e) Site name: airship-seaworthy
 
-2. Spyglass CLI Command:
+2. Spyglass CLI Command using an Excel File:
 
 .. code-block:: bash
 
-    spyglass m -i -p tugboat -x SiteDesignSpec_v0.1.xlsx \
+    spyglass excel documents -i \
-               -e excel_spec_upstream.yaml -c site_config.yaml \
+        -x ../spyglass-plugin-xls/spyglass_plugin_xls/examples/SiteDesignSpec_v0.1.xlsx \
-               -s airship-seaworthy -t <relative path to J2 templates dir>
+        -e ../spyglass-plugin-xls/spyglass_plugin_xls/examples/excel_spec.yaml \
+        -c ../spyglass-plugin-xls/spyglass_plugin_xls/examples/site_config.yaml \
+        -s airship-seaworthy -t spyglass/examples/templates/
+
+3. Spyglass CLI Command using a CSV File:
+
+.. code-block:: bash
+
+    spyglass excel documents -i \
+        -x ../spyglass-plugin-xls/spyglass_plugin_xls/examples/Site-Information.csv \
+        -e ../spyglass-plugin-xls/spyglass_plugin_xls/examples/excel_spec.yaml \
+        -c ../spyglass-plugin-xls/spyglass_plugin_xls/examples/site_config.yaml \
+        -s airship-seaworthy -t spyglass/examples/templates/

spyglass_plugin_xls/examples/Site-Information.csv

@@ -0,0 +1,63 @@
+Site-Information,,,,
+IPMI Host Details,,,,
+,Server Name,IPMI Address,IPMI Gateway,Host Profile
+,cab2r72c12,10.0.220.138,10.0.220.129,dp-r720
+,cab2r72c13,10.0.220.139,10.0.220.129,dp-r720
+,cab2r72c14,10.0.220.140,10.0.220.129,dp-r720
+,cab2r72c15,10.0.220.141,10.0.220.129,dp-r720
+,cab2r72c16,10.0.220.142,10.0.220.129,cp-r720
+,cab2r72c17,10.0.220.143,10.0.220.129,cp-r720
+,cab2r73c12,10.0.220.170,10.0.220.161,dp-r720
+,cab2r73c13,10.0.220.171,10.0.220.161,dp-r720
+,cab2r73c14,10.0.220.172,10.0.220.161,dp-r720
+,cab2r73c15,10.0.220.173,10.0.220.161,dp-r720
+,cab2r73c16,10.0.220.174,10.0.220.161,cp-r720
+,cab2r73c17,10.0.220.175,10.0.220.161,cp-r720
+,,,,
+Network Details,,,,
+Usage,vLAN,,,
+iSCSI/Storage,vLAN 23,,,
+PXE,vLAN 21,,,
+Calico BGP peering addresses,vLAN 22,,,
+Overlay,vLAN 24,,,
+CNI Pod addresses,N/A,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+VLAN,POD,,,
+vLAN 23,30.31.1.0/25,,,
+vLAN 21,30.30.4.0/25,,,
+vLAN 21,30.30.4.128/25,,,
+vLAN 21,30.30.5.0/25,,,
+vLAN 21,30.30.5.128/25,,,
+,,,,
+vLAN 22,30.29.1.0/25,,,
+vLAN 24,30.19.0.0/25,,,
+,,,,
+VLAN,Rack1,Rack2,Rack3,Rack4
+VLAN-21,10.0.220.0/26,,,
+VLAN-49,10.0.220.64/29,,,
+Calico Ingress,10.0.220.72/29,,,
+Loopback 3 ,10.0.220.80/27,,,
+Spare,10.0.220.112-10.0.220.127,,,
+VLAN 1023,10.0.220.128/27,10.0.220.160/27,10.0.220.192/27,10.0.220.224/27
+SiteDetails,,,,
+,,,,
+Domain,dmy00.example.com,,,
+Subdomain,testitservices,,,
+Group,AA-AAA-dmy00,,,
+ldap_url,url: ldap://ldap.example.com,,,
+NTP,150.234.210.5 (ns1.example.com),,,
+DNS,"40.40.40.40 (ntp1.example.com),
+41.41.41.41 (ntp2.example.com)",,,
+,,,,
+Sitename,SampleSiteName,,,
+Corridor,Corridor 1,,,
+State,New Jersey,,,
+Country,SampleCountry,,,
+Clli,XXXXXX21,,,

spyglass_plugin_xls/excel_parser.py

@@ -13,7 +13,9 @@
 # limitations under the License.
 
 from copy import deepcopy
+import csv
 import logging
+import os
 import pprint
 import re
 
@@ -316,9 +318,18 @@ class ExcelParser(object):
 
     @staticmethod
     def load_excel_data(filename):
-        """Combines multiple excel file to a single design spec"""
+        """Combines multiple excel or csv files to a single design spec"""
 
         design_spec = Workbook()
+        if os.path.splitext(filename)[1] == '.csv':
+            loaded_workbook = Workbook()
+            ws = loaded_workbook.active
+            with open(filename) as f:
+                reader = csv.reader(f, delimiter=',')
+                for row in reader:
+                    ws.append(row)
+            ws.title = os.path.splitext(os.path.basename(filename))[0]
+        else:
             loaded_workbook = load_workbook(filename, data_only=True)
         for names in loaded_workbook.sheetnames:
             design_spec_worksheet = design_spec.create_sheet(names)
@@ -334,11 +345,21 @@ class ExcelParser(object):
 
         'MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs'
         """
-
+        file_type = os.path.splitext(sheetname.split(':')[0])[1]
-        if re.search(".xlsx", sheetname) or re.search(".xls", sheetname):
+        if file_type == '.xlsx' or file_type == '.xls':
             # Extract file name
             source_xl_file = sheetname.split(":")[0]
             wb = load_workbook(source_xl_file, data_only=True)
             return [wb, sheetname.split(":")[1]]
+        elif file_type == ".csv":
+            source_csv_file = sheetname.split(":")[0]
+            wb = Workbook()
+            ws = wb.active
+            with open(source_csv_file) as f:
+                reader = csv.reader(f, delimiter=',')
+                for row in reader:
+                    ws.append(row)
+            ws.title = os.path.splitext(os.path.basename(source_csv_file))[0]
+            return [wb, sheetname.split(":")[1]]
         else:
             return [None, sheetname]

tests/shared/Site-Information.csv

@@ -0,0 +1,63 @@
+Site-Information,,,,
+IPMI Host Details,,,,
+,Server Name,IPMI Address,IPMI Gateway,Host Profile
+,cab2r72c12,10.0.220.138,10.0.220.129,dp-r720
+,cab2r72c13,10.0.220.139,10.0.220.129,dp-r720
+,cab2r72c14,10.0.220.140,10.0.220.129,dp-r720
+,cab2r72c15,10.0.220.141,10.0.220.129,dp-r720
+,cab2r72c16,10.0.220.142,10.0.220.129,cp-r720
+,cab2r72c17,10.0.220.143,10.0.220.129,cp-r720
+,cab2r73c12,10.0.220.170,10.0.220.161,dp-r720
+,cab2r73c13,10.0.220.171,10.0.220.161,dp-r720
+,cab2r73c14,10.0.220.172,10.0.220.161,dp-r720
+,cab2r73c15,10.0.220.173,10.0.220.161,dp-r720
+,cab2r73c16,10.0.220.174,10.0.220.161,cp-r720
+,cab2r73c17,10.0.220.175,10.0.220.161,cp-r720
+,,,,
+Network Details,,,,
+Usage,vLAN,,,
+iSCSI/Storage,vLAN 23,,,
+PXE,vLAN 21,,,
+Calico BGP peering addresses,vLAN 22,,,
+Overlay,vLAN 24,,,
+CNI Pod addresses,N/A,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+VLAN,POD,,,
+vLAN 23,30.31.1.0/25,,,
+vLAN 21,30.30.4.0/25,,,
+vLAN 21,30.30.4.128/25,,,
+vLAN 21,30.30.5.0/25,,,
+vLAN 21,30.30.5.128/25,,,
+,,,,
+vLAN 22,30.29.1.0/25,,,
+vLAN 24,30.19.0.0/25,,,
+,,,,
+VLAN,Rack1,Rack2,Rack3,Rack4
+VLAN-21,10.0.220.0/26,,,
+VLAN-49,10.0.220.64/29,,,
+Calico Ingress,10.0.220.72/29,,,
+Loopback 3 ,10.0.220.80/27,,,
+Spare,10.0.220.112-10.0.220.127,,,
+VLAN 1023,10.0.220.128/27,10.0.220.160/27,10.0.220.192/27,10.0.220.224/27
+SiteDetails,,,,
+,,,,
+Domain,dmy00.example.com,,,
+Subdomain,testitservices,,,
+Group,AA-AAA-dmy00,,,
+ldap_url,url: ldap://ldap.example.com,,,
+NTP,150.234.210.5 (ns1.example.com),,,
+DNS,"40.40.40.40 (ntp1.example.com),
+41.41.41.41 (ntp2.example.com)",,,
+,,,,
+Sitename,SampleSiteName,,,
+Corridor,Corridor 1,,,
+State,New Jersey,,,
+Country,SampleCountry,,,
+Clli,XXXXXX21,,,

tests/unit/test_excel_parser.py

@@ -32,6 +32,8 @@ INVALID_EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'invalid_excel_spec.yaml')
 
 EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx')
 
+CSV_FILE_PATH = os.path.join(FIXTURE_DIR, "Site-Information.csv")
+
 SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml')
 
 
@@ -55,6 +57,18 @@ class TestExcelParser(unittest.TestCase):
         self.assertIsInstance(result.wb_combined, Workbook)
         self.assertEqual('xl_spec', result.spec)
 
+    def test___init__csv(self):
+        with open(EXCEL_SPEC_PATH, 'r') as f:
+            loaded_spec = yaml.safe_load(f)
+        result = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+        self.assertEqual(CSV_FILE_PATH, result.file_name)
+        self.assertDictEqual(loaded_spec, result.excel_specs)
+        self.assertDictEqual(
+            loaded_spec['specs'][self.SPEC], result.loaded_spec)
+        self.assertDictEqual(self.raw_excel_data, result.loaded_data)
+        self.assertIsInstance(result.wb_combined, Workbook)
+        self.assertEqual('xl_spec', result.spec)
+
     def test_sanitize(self):
         test_string = 'Hello THIS is A TeSt'
         expected_output = 'hellothisisatest'
@@ -68,6 +82,13 @@ class TestExcelParser(unittest.TestCase):
         result = obj.compare(test_string1, test_string2)
         self.assertTrue(result)
 
+    def test_compare_csv(self):
+        test_string1 = 'These strings are equal.'
+        test_string2 = 'These strIngs are Equal .'
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.compare(test_string1, test_string2)
+        self.assertTrue(result)
+
     def test_compare_false(self):
         test_string1 = 'These strings are not equal.'
         test_string2 = 'These strIngs are Equal.'
@@ -75,11 +96,23 @@ class TestExcelParser(unittest.TestCase):
         result = obj.compare(test_string1, test_string2)
         self.assertFalse(result)
 
+    def test_compare_false_csv(self):
+        test_string1 = 'These strings are not equal.'
+        test_string2 = 'These strIngs are Equal.'
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.compare(test_string1, test_string2)
+        self.assertFalse(result)
+
     def test__get_workbook(self):
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
         result = obj._get_workbook('Site-Information')
         self.assertIsInstance(result, Worksheet)
 
+    def test__get_workbook_csv(self):
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj._get_workbook('Site-Information')
+        self.assertIsInstance(result, Worksheet)
+
     def test__check_sanitize_settings(self):
         test_data_sanitize_only = {'sanitize': False}
         sanitize, no_sanitize_keys = ExcelParser._check_sanitize_settings(
@@ -102,30 +135,61 @@ class TestExcelParser(unittest.TestCase):
             obj.loaded_spec['public']['data']['oam'], 'Site-Information')
         self.assertDictEqual(expected_data, result)
 
+    def test_extract_data_points_csv(self):
+        expected_data = self.raw_excel_data['public']['oam']
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+        result = obj.extract_data_points(
+            obj.loaded_spec['public']['data']['oam'], 'Site-Information')
+        self.assertDictEqual(expected_data, result)
+
     def test_extract_data_points_unsanitized(self):
         expected_data = self.raw_excel_data['location']
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
         result = obj.extract_data_points(obj.loaded_spec['location'])
         self.assertDictEqual(expected_data, result)
 
+    def test_extract_data_points_unsanitized_csv(self):
+        expected_data = self.raw_excel_data['location']
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+        result = obj.extract_data_points(obj.loaded_spec['location'])
+        self.assertDictEqual(expected_data, result)
+
     def test_extract_data_series(self):
         expected_data = self.raw_excel_data['ipmi']
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
         result = obj.extract_data_series(obj.loaded_spec['ipmi'])
         self.assertEqual(expected_data, result)
 
+    def test_extract_data_series_csv(self):
+        expected_data = self.raw_excel_data['ipmi']
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+        result = obj.extract_data_series(obj.loaded_spec['ipmi'])
+        self.assertEqual(expected_data, result)
+
     def test_extract_data_series_no_sanitize(self):
         expected_data = self.raw_excel_data['private_vlan']
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
         result = obj.extract_data_series(obj.loaded_spec['private_vlan'])
         self.assertEqual(expected_data, result)
 
+    def test_extract_data_series_no_sanitize_csv(self):
+        expected_data = self.raw_excel_data['private_vlan']
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+        result = obj.extract_data_series(obj.loaded_spec['private_vlan'])
+        self.assertEqual(expected_data, result)
+
     def test_extract_data_using_spec(self):
         expected_data = self.raw_excel_data
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
         result = obj.extract_data_using_spec(obj.loaded_spec)
         self.assertDictEqual(expected_data, result)
 
+    def test_extract_data_using_spec_csv(self):
+        expected_data = self.raw_excel_data
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+        result = obj.extract_data_using_spec(obj.loaded_spec)
+        self.assertDictEqual(expected_data, result)
+
     def test_get_ipmi_data(self):
         expected_hosts = self.site_data['ipmi_data'][1]
         expected_ipmi_data = self.site_data['ipmi_data'][0]
@@ -134,6 +198,14 @@ class TestExcelParser(unittest.TestCase):
         self.assertDictEqual(result[0], expected_ipmi_data)
         self.assertEqual(result[1], expected_hosts)
 
+    def test_get_ipmi_data_csv(self):
+        expected_hosts = self.site_data['ipmi_data'][1]
+        expected_ipmi_data = self.site_data['ipmi_data'][0]
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_ipmi_data()
+        self.assertDictEqual(result[0], expected_ipmi_data)
+        self.assertEqual(result[1], expected_hosts)
+
     def test_get_private_vlan_data(self):
         expected_vlan_data = {
             'vlan23': 'iSCSI/Storage',
@@ -146,48 +218,102 @@ class TestExcelParser(unittest.TestCase):
         result = obj.get_private_vlan_data()
         self.assertDictEqual(expected_vlan_data, result)
 
+    def test_get_private_vlan_data_csv(self):
+        expected_vlan_data = {
+            'vlan23': 'iSCSI/Storage',
+            'vlan21': 'PXE',
+            'vlan22': 'Calico BGP peering addresses',
+            'vlan24': 'Overlay',
+            'na': 'CNI Pod addresses'
+        }
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_private_vlan_data()
+        self.assertDictEqual(expected_vlan_data, result)
+
     def test_get_private_network_data(self):
         expected_network_data = self.site_data['network_data']['private']
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
         result = obj.get_private_network_data()
         self.assertDictEqual(expected_network_data, result)
 
+    def test_get_private_network_data_csv(self):
+        expected_network_data = self.site_data['network_data']['private']
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_private_network_data()
+        self.assertDictEqual(expected_network_data, result)
+
     def test_get_public_network_data(self):
         expected_network_data = self.site_data['network_data']['public']
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
         result = obj.get_public_network_data()
         self.assertEqual(expected_network_data, result)
 
+    def test_get_public_network_data_csv(self):
+        expected_network_data = self.site_data['network_data']['public']
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_public_network_data()
+        self.assertEqual(expected_network_data, result)
+
     def test_get_site_info(self):
         expected_site_info = self.site_data['site_info']
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
         result = obj.get_site_info()
         self.assertDictEqual(expected_site_info, result)
 
+    def test_get_site_info_csv(self):
+        expected_site_info = self.site_data['site_info']
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_site_info()
+        self.assertDictEqual(expected_site_info, result)
+
     def test_get_location_data(self):
         expected_location_data = self.site_data['site_info']['location']
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
         result = obj.get_location_data()
         self.assertEqual(expected_location_data, result)
 
+    def test_get_location_data_csv(self):
+        expected_location_data = self.site_data['site_info']['location']
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_location_data()
+        self.assertEqual(expected_location_data, result)
+
     def test_validate_sheet_names_with_spec(self):
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
         self.assertIsNone(obj.validate_sheet_names_with_spec())
 
+    def test_validate_sheet_names_with_spec_csv(self):
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        self.assertIsNone(obj.validate_sheet_names_with_spec())
+
     def test_validate_sheet_names_with_spec_invalid(self):
         with self.assertRaises(ExcelSheetNotFound):
             ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
 
+    def test_validate_sheet_names_with_spec_invalid_csv(self):
+        with self.assertRaises(ExcelSheetNotFound):
+            ExcelParser(CSV_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
+
     def test_get_data(self):
         expected_data = self.site_data
         obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
         result = obj.get_data()
         self.assertDictEqual(expected_data, result)
 
+    def test_get_data_csv(self):
+        expected_data = self.site_data
+        obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+        result = obj.get_data()
+        self.assertDictEqual(expected_data, result)
+
     def test_load_excel_data(self):
         result = ExcelParser.load_excel_data(EXCEL_FILE_PATH)
         self.assertIsInstance(result, Workbook)
 
+    def test_load_csv_data(self):
+        result = ExcelParser.load_excel_data(CSV_FILE_PATH)
+        self.assertIsInstance(result, Workbook)
+
     def test_get_xl_obj_and_sheetname(self):
         result = ExcelParser.get_xl_obj_and_sheetname('Site-Information')
         self.assertEqual([None, 'Site-Information'], result)
@@ -198,3 +324,10 @@ class TestExcelParser(unittest.TestCase):
         self.assertIsInstance(result, list)
         self.assertIsInstance(result[0], Workbook)
         self.assertEqual(result[1], 'Site-Information')
+
+    def test_get_xl_obj_and_sheetname_file_specified_csv(self):
+        sheet = CSV_FILE_PATH + ':Site-Information'
+        result = ExcelParser.get_xl_obj_and_sheetname(sheet)
+        self.assertIsInstance(result, list)
+        self.assertIsInstance(result[0], Workbook)
+        self.assertEqual(result[1], 'Site-Information')