Add unit tests for xls plugin CLI

This change implements unit tests for the Spyglass XLS plugin CLI.

Change-Id: I0761f8508f3134b1ffb77e78543a72bf1fcf5ca5

.zuul.yaml

@@ -1,3 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 - project:
     templates:
-      - noop-jobs
+      - openstack-python36-jobs
+      - openstack-python37-jobs
+      - openstack-cover-jobs
+    check:
+      jobs:
+        - openstack-tox-pep8
+    gate:
+      jobs:
+        - openstack-tox-pep8

spyglass_plugin_xls/cli.py

@@ -151,7 +151,7 @@ def generate_manifests_and_intermediary(*args, **kwargs):
         spyglass.cli.intermediary_processor('excel', **kwargs)
     LOG.info("Generate intermediary yaml")
     intermediary_yaml = process_input_ob.generate_intermediary_yaml()
-    if generate_intermediary:
+    if kwargs["generate_intermediary"]:
         LOG.debug("Dumping intermediary yaml")
         process_input_ob.dump_intermediary_file(kwargs['intermediary_dir'])
     else:

spyglass_plugin_xls/examples/excel_spec.yaml

@@ -17,47 +17,47 @@
 # design spec file.
 ---
 specs:
-    # Design Spec file name: SiteDesignSpec_v0.1.xlsx
-    xl_spec:
-        ipmi_sheet_name: 'Site-Information'
-        start_row: 4
-        end_row: 15
-        hostname_col: 2
-        ipmi_address_col: 3
-        host_profile_col: 5
-        ipmi_gateway_col: 4
-        private_ip_sheet: 'Site-Information'
-        net_type_col: 1
-        vlan_col: 2
-        vlan_start_row: 19
-        vlan_end_row: 30
-        net_start_row: 33
-        net_end_row: 40
-        net_col: 2
-        net_vlan_col: 1
-        public_ip_sheet: 'Site-Information'
-        oam_vlan_col: 1
-        oam_ip_row: 43
-        oam_ip_col: 2
-        oob_net_row: 48
-        oob_net_start_col: 2
-        oob_net_end_col: 5
-        ingress_ip_row: 45
-        dns_ntp_ldap_sheet: 'Site-Information'
-        login_domain_row: 52
-        ldap_col: 2
-        global_group: 53
-        ldap_search_url_row: 54
-        ntp_row: 55
-        ntp_col: 2
-        dns_row: 56
-        dns_col: 2
-        domain_row: 51
-        domain_col: 2
-        location_sheet: 'Site-Information'
-        column: 2
-        corridor_row: 59
-        site_name_row: 58
-        state_name_row: 60
-        country_name_row: 61
-        clli_name_row: 62
+  # Design Spec file name: SiteDesignSpec_v0.1.xlsx
+  xl_spec:
+    ipmi_sheet_name: 'Site-Information'
+    start_row: 4
+    end_row: 15
+    hostname_col: 2
+    ipmi_address_col: 3
+    host_profile_col: 5
+    ipmi_gateway_col: 4
+    private_ip_sheet: 'Site-Information'
+    net_type_col: 1
+    vlan_col: 2
+    vlan_start_row: 19
+    vlan_end_row: 30
+    net_start_row: 33
+    net_end_row: 40
+    net_col: 2
+    net_vlan_col: 1
+    public_ip_sheet: 'Site-Information'
+    oam_vlan_col: 1
+    oam_ip_row: 43
+    oam_ip_col: 2
+    oob_net_row: 48
+    oob_net_start_col: 2
+    oob_net_end_col: 5
+    ingress_ip_row: 45
+    dns_ntp_ldap_sheet: 'Site-Information'
+    login_domain_row: 52
+    ldap_col: 2
+    global_group: 53
+    ldap_search_url_row: 54
+    ntp_row: 55
+    ntp_col: 2
+    dns_row: 56
+    dns_col: 2
+    domain_row: 51
+    domain_col: 2
+    location_sheet: 'Site-Information'
+    column: 2
+    corridor_row: 59
+    site_name_row: 58
+    state_name_row: 60
+    country_name_row: 61
+    clli_name_row: 62

spyglass_plugin_xls/examples/site_config.yaml

@@ -1,6 +1,6 @@
-##################################
-# Site Specific Tugboat Settings #
-##################################
+##############################################
+# Site Specific Spyglass XLS Plugin Settings #
+##############################################
 ---
 site_info:
   ldap:

test-requirements.txt

@@ -1,3 +1,8 @@
+# Testing
+pytest==4.4.1
+pytest-xdist==1.28.0
+pytest-cov==2.6.1
+
 # Formatting
 yapf==0.27.0
 

tests/shared/excel_spec.yaml

@@ -0,0 +1,63 @@
+# Copyright 2018 The Openstack-Helm Authors.
+# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Important: Please modify the dictionary with appropriate
+# design spec file.
+---
+specs:
+  # Design Spec file name: SiteDesignSpec_v0.1.xlsx
+  xl_spec:
+    ipmi_sheet_name: 'Site-Information'
+    start_row: 4
+    end_row: 15
+    hostname_col: 2
+    ipmi_address_col: 3
+    host_profile_col: 5
+    ipmi_gateway_col: 4
+    private_ip_sheet: 'Site-Information'
+    net_type_col: 1
+    vlan_col: 2
+    vlan_start_row: 19
+    vlan_end_row: 30
+    net_start_row: 33
+    net_end_row: 40
+    net_col: 2
+    net_vlan_col: 1
+    public_ip_sheet: 'Site-Information'
+    oam_vlan_col: 1
+    oam_ip_row: 43
+    oam_ip_col: 2
+    oob_net_row: 48
+    oob_net_start_col: 2
+    oob_net_end_col: 5
+    ingress_ip_row: 45
+    dns_ntp_ldap_sheet: 'Site-Information'
+    login_domain_row: 52
+    ldap_col: 2
+    global_group: 53
+    ldap_search_url_row: 54
+    ntp_row: 55
+    ntp_col: 2
+    dns_row: 56
+    dns_col: 2
+    domain_row: 51
+    domain_col: 2
+    location_sheet: 'Site-Information'
+    column: 2
+    corridor_row: 59
+    site_name_row: 58
+    state_name_row: 60
+    country_name_row: 61
+    clli_name_row: 62

tests/shared/site_config.yaml

@@ -0,0 +1,33 @@
+##############################################
+# Site Specific Spyglass XLS Plugin Settings #
+##############################################
+---
+site_info:
+  ldap:
+    common_name: test
+    url: ldap://ldap.example.com
+    subdomain: test
+  ntp:
+    servers: 10.10.10.10,20.20.20.20,30.30.30.30
+  sitetype: foundry
+  domain: atlantafoundry.com
+  dns:
+    servers: 8.8.8.8,8.8.4.4,208.67.222.222
+network:
+  vlan_network_data:
+    ingress:
+        subnet:
+        - 132.68.226.72/29
+  bgp :
+    peers:
+    - '172.29.0.2'
+    - '172.29.0.3'
+    asnumber: 64671
+    peer_asnumber: 64688
+storage:
+  ceph:
+    controller:
+      osd_count: 6
+...
+
+

tests/shared/templates/site-definition.yaml.j2

@@ -0,0 +1,13 @@
+---
+schema: pegleg/SiteDefinition/v1
+metadata:
+  schema: metadata/Document/v1
+  layeringDefinition:
+    abstract: false
+    layer: site
+  name: {{ data['region_name'] }}
+  storagePolicy: cleartext
+data:
+  site_type: {{ data['site_info']['sitetype'] }}
+...
+

tests/unit/test_cli.py

@@ -0,0 +1,229 @@
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from unittest import mock
+
+from click.testing import CliRunner
+from spyglass.site_processors.site_processor import SiteProcessor
+
+from spyglass_plugin_xls.cli import generate_intermediary, \
+    generate_manifests_and_intermediary
+
+FIXTURE_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(__file__)), 'shared')
+
+EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'excel_spec.yaml')
+
+EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx')
+
+SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml')
+
+TEMPLATE_PATH = os.path.join(FIXTURE_DIR, 'templates')
+
+
+@mock.patch('spyglass.cli.intermediary_processor', autospec=True)
+@mock.patch('spyglass_plugin_xls.excel.ExcelPlugin', autospec=False)
+def test_generate_intermediary(mock_excel_plugin, mock_intermediary_processor):
+    """Test generate_intermediary (intermediary) normal execution"""
+    runner = CliRunner()
+    args = [
+        '-x',
+        EXCEL_FILE_PATH,
+        '-e',
+        EXCEL_SPEC_PATH,
+        '-c',
+        SITE_CONFIG_PATH,
+        '-s',
+        'airship-seaworthy'
+    ]
+    result = runner.invoke(generate_intermediary, args)
+    assert result.exit_code == 0
+    mock_intermediary_processor.assert_called_once()
+    mock_intermediary_processor.return_value.generate_intermediary_yaml\
+        .assert_called_once()
+    mock_intermediary_processor.return_value.dump_intermediary_file\
+        .assert_called_once()
+
+
+@mock.patch('spyglass.cli.intermediary_processor', autospec=True)
+@mock.patch('spyglass_plugin_xls.excel.ExcelPlugin', autospec=False)
+def test_generate_intermediary_intermediary_dir(
+        mock_excel_plugin, mock_intermediary_processor, tmpdir):
+    """Test intermediary_directory option for intermediary"""
+    runner = CliRunner()
+    test_dir_name = 'intermediary_test_dir'
+    test_dir = tmpdir.mkdir(test_dir_name)
+    args = [
+        '-x',
+        EXCEL_FILE_PATH,
+        '-e',
+        EXCEL_SPEC_PATH,
+        '-c',
+        SITE_CONFIG_PATH,
+        '-s',
+        'airship-seaworthy',
+        '--intermediary-dir',
+        test_dir
+    ]
+    result = runner.invoke(generate_intermediary, args)
+    assert result.exit_code == 0
+    mock_intermediary_processor.assert_called_once()
+    mock_intermediary_processor.return_value.generate_intermediary_yaml\
+        .assert_called_once()
+    mock_intermediary_processor.return_value.dump_intermediary_file\
+        .assert_called_once_with(test_dir)
+
+
+@mock.patch('spyglass.cli.intermediary_processor', autospec=True)
+@mock.patch('spyglass_plugin_xls.excel.ExcelPlugin', autospec=False)
+def test_generate_manifests_and_intermediary(
+        mock_excel_plugin, mock_intermediary_processor):
+    """Test generate_manifests_and_intermediary (documents) normal execution"""
+    runner = CliRunner()
+    args = [
+        '-x',
+        EXCEL_FILE_PATH,
+        '-e',
+        EXCEL_SPEC_PATH,
+        '-c',
+        SITE_CONFIG_PATH,
+        '-s',
+        'airship-seaworthy',
+        '-t',
+        TEMPLATE_PATH
+    ]
+    with mock.patch.object(
+            SiteProcessor,
+            'render_template',
+            autospec=True) as mock_render_template:
+        result = runner.invoke(generate_manifests_and_intermediary, args)
+    assert result.exit_code == 0
+    mock_intermediary_processor.assert_called_once()
+    mock_intermediary_processor.return_value.generate_intermediary_yaml\
+        .assert_called_once()
+    assert not mock_intermediary_processor.return_value.dump_intermediary_file\
+        .called
+    mock_render_template.assert_called_once_with(mock.ANY, TEMPLATE_PATH)
+
+
+@mock.patch('spyglass.cli.intermediary_processor', autospec=True)
+@mock.patch('spyglass_plugin_xls.excel.ExcelPlugin', autospec=False)
+def test_generate_manifests_and_intermediary_generate_intermediary(
+        mock_excel_plugin, mock_intermediary_processor):
+    """Test generate_intermediary option for documents command"""
+    runner = CliRunner()
+    args = [
+        '-x',
+        EXCEL_FILE_PATH,
+        '-e',
+        EXCEL_SPEC_PATH,
+        '-c',
+        SITE_CONFIG_PATH,
+        '-s',
+        'airship-seaworthy',
+        '-t',
+        TEMPLATE_PATH,
+        '-i'
+    ]
+    with mock.patch.object(
+            SiteProcessor,
+            'render_template',
+            autospec=True) as mock_render_template:
+        result = runner.invoke(generate_manifests_and_intermediary, args)
+    assert result.exit_code == 0
+    mock_intermediary_processor.assert_called_once()
+    mock_intermediary_processor.return_value.generate_intermediary_yaml\
+        .assert_called_once()
+    assert mock_intermediary_processor.return_value.dump_intermediary_file\
+        .called
+    mock_render_template.assert_called_once_with(mock.ANY, TEMPLATE_PATH)
+
+
+@mock.patch('spyglass.cli.intermediary_processor', autospec=True)
+@mock.patch('spyglass_plugin_xls.excel.ExcelPlugin', autospec=False)
+def test_generate_manifests_and_intermediary_intermediary_dir(
+        mock_excel_plugin, mock_intermediary_processor, tmpdir):
+    """Test intermediary_dir option for documents command"""
+    runner = CliRunner()
+    test_dir_name = 'intermediary_test_dir'
+    test_dir = tmpdir.mkdir(test_dir_name)
+    args = [
+        '-x',
+        EXCEL_FILE_PATH,
+        '-e',
+        EXCEL_SPEC_PATH,
+        '-c',
+        SITE_CONFIG_PATH,
+        '-s',
+        'airship-seaworthy',
+        '-t',
+        TEMPLATE_PATH,
+        '-i',
+        '--intermediary-dir',
+        test_dir
+    ]
+    with mock.patch.object(
+            SiteProcessor,
+            'render_template',
+            autospec=True) as mock_render_template:
+        result = runner.invoke(generate_manifests_and_intermediary, args)
+    assert result.exit_code == 0
+    mock_intermediary_processor.assert_called_once()
+    mock_intermediary_processor.return_value.generate_intermediary_yaml\
+        .assert_called_once()
+    mock_intermediary_processor.return_value.dump_intermediary_file\
+        .assert_called_once_with(test_dir)
+    mock_render_template.assert_called_once_with(mock.ANY, TEMPLATE_PATH)
+
+
+@mock.patch('spyglass.cli.intermediary_processor', autospec=True)
+@mock.patch('spyglass_plugin_xls.excel.ExcelPlugin', autospec=False)
+@mock.patch.object(
+    SiteProcessor, '__init__', autospec=True, return_value=None)
+def test_generate_manifests_and_intermediary_manifest_dir(
+        mock_site_processor, mock_excel_plugin, mock_intermediary_processor,
+        tmpdir):
+    """Test manifest_dir option for documents command"""
+    runner = CliRunner()
+    test_dir_name = 'manifest_test_dir'
+    test_dir = tmpdir.mkdir(test_dir_name)
+    args = [
+        '-x',
+        EXCEL_FILE_PATH,
+        '-e',
+        EXCEL_SPEC_PATH,
+        '-c',
+        SITE_CONFIG_PATH,
+        '-s',
+        'airship-seaworthy',
+        '-t',
+        TEMPLATE_PATH,
+        '--manifest-dir',
+        test_dir
+    ]
+    with mock.patch.object(
+            SiteProcessor,
+            'render_template',
+            autospec=True) as mock_render_template:
+        result = runner.invoke(generate_manifests_and_intermediary, args)
+    assert result.exit_code == 0
+    mock_intermediary_processor.assert_called_once()
+    mock_intermediary_processor.return_value.generate_intermediary_yaml\
+        .assert_called_once()
+    assert not mock_intermediary_processor.return_value.dump_intermediary_file\
+        .called
+    mock_site_processor.assert_called_once_with(
+        mock.ANY, mock.ANY, test_dir, False)
+    mock_render_template.assert_called_once_with(mock.ANY, TEMPLATE_PATH)

tools/gate/run-unit-tests.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+posargs=$@
+# cross-platform way to derive the number of logical cores
+readonly num_cores=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())')
+if [ ${#posargs} -ge 1 ]; then
+    pytest -k ${posargs} -n $num_cores
+else
+    pytest -n $num_cores
+fi
+set +e

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = pep8, docs
+envlist = py36, py37, pep8, docs, cover
 minversion = 2.3.1
 skipsdist = True
 
@@ -63,3 +63,15 @@ commands =
     rm -rf doc/build
     sphinx-build -b html doc/source doc/build -n -W -v
 whitelist_externals = rm
+
+[testenv:cover]
+basepython = python3
+deps =
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/test-requirements.txt
+commands =
+  bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=spyglass_plugin_xls \
+      --cov-report html:cover --cov-report xml:cover/coverage.xml \
+      --cov-report term --cov-fail-under 20 tests/'
+whitelist_externals =
+  bash