Base code for spyglass

- Spyglass skelton with engine, site processor
- Spyglass data extractor with formation plugin
- Docker files and scripts to run spyglass

.dockerignore

@@ -0,0 +1,4 @@
+**/__pycache__
+**/.tox
+**/.eggs
+**/spyglass.egg-info

.gitignore

@@ -0,0 +1,116 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+*.tgz
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.testrepository/*
+cover/*
+results/*
+.stestr/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# pycharm-ide
+.idea/
+
+# osx
+.DS_Store
+
+# git
+Changelog
+AUTHORS
+
+# Ansible
+*.retry

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,84 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SPYGLASS_BUILD_CTX  ?= .
+IMAGE_NAME        ?= spyglass
+IMAGE_PREFIX      ?= att-comdev
+DOCKER_REGISTRY   ?= quay.io
+IMAGE_TAG         ?= latest
+PROXY             ?= http://proxy.foo.com:8000
+NO_PROXY          ?= localhost,127.0.0.1,.svc.cluster.local
+USE_PROXY         ?= false
+PUSH_IMAGE        ?= false
+LABEL             ?= commit-id
+IMAGE             ?= $(DOCKER_REGISTRY)/$(IMAGE_PREFIX)/$(IMAGE_NAME):$(IMAGE_TAG)
+PYTHON_BASE_IMAGE ?= python:3.6
+export
+
+# Build all docker images for this project
+.PHONY: images
+images: build_spyglass
+
+# Run an image locally and exercise simple tests
+.PHONY: run_images
+run_images: run_spyglass
+
+.PHONY: run_spyglass
+run_spyglass: build_spyglass
+	tools/spyglass.sh --help
+
+.PHONY: security
+security:
+	tox -c tox.ini -e bandit
+
+# Perform Linting
+.PHONY: lint
+lint: py_lint
+
+# Perform auto formatting
+.PHONY: format
+format: py_format
+
+.PHONY: build_spyglass
+build_spyglass:
+ifeq ($(USE_PROXY), true)
+	docker build -t $(IMAGE) --network=host --label $(LABEL) -f images/spyglass/Dockerfile \
+		--build-arg FROM=$(PYTHON_BASE_IMAGE) \
+		--build-arg http_proxy=$(PROXY) \
+		--build-arg https_proxy=$(PROXY) \
+		--build-arg HTTP_PROXY=$(PROXY) \
+		--build-arg HTTPS_PROXY=$(PROXY) \
+		--build-arg no_proxy=$(NO_PROXY) \
+		--build-arg NO_PROXY=$(NO_PROXY) \
+		--build-arg ctx_base=$(SPYGLASS_BUILD_CTX) .
+else
+	docker build -t $(IMAGE) --network=host --label $(LABEL) -f images/spyglass/Dockerfile \
+		--build-arg FROM=$(PYTHON_BASE_IMAGE) \
+		--build-arg ctx_base=$(SPYGLASS_BUILD_CTX) .
+endif
+ifeq ($(PUSH_IMAGE), true)
+	docker push $(IMAGE)
+endif
+
+.PHONY: clean
+clean:
+	rm -rf build
+
+.PHONY: py_lint
+py_lint:
+	tox -e pep8
+
+.PHONY: py_format
+py_format:
+	tox -e fmt

README.md

@@ -1,2 +1,29 @@
-# spyglass
-staging for the spyglass airship-spyglass repo
+
+What is Spyglass?
+----------------
+
+Spyglass is the data extractor tool which can interface with
+different input data sources to generate site manifest YAML files.
+The data sources will provide all the configuration data needed
+for a site deployment. These site manifest YAML files generated
+by spyglass will be saved in a Git repository, from where Pegleg
+can access and aggregate them. This aggregated file can then be
+fed to shipyard for site deployment / updates.
+
+Spyglass follows plugin model to support multiple input data sources.
+Current supported plugins are formation-plugin and Tugboat. Formation
+is a rest API based service which will be the source of information
+related to hardware, networking, site data. Formation plugin will
+interact with Formation API to gather necessary configuration.
+Similarly Tugboat accepts engineering spec which is in the form of
+spreadsheet and an index file to read spreadsheet as inputs and
+generates the site level manifests.
+As an optional step it can generate an intermediary yaml which contain
+all the information that will be rendered to generate Airship site
+manifests. This optional step will help the deployment engineer to
+modify any data if required.
+
+Basic Usage
+-----------
+
+TODO

images/spyglass/Dockerfile

@@ -0,0 +1,13 @@
+ARG FROM=python:3.6
+FROM ${FROM}
+
+VOLUME /var/spyglass
+WORKDIR /var/spyglass
+
+ARG ctx_base=.
+
+COPY ${ctx_base}/requirements.txt /opt/spyglass/requirements.txt
+RUN pip3 install --no-cache-dir -r /opt/spyglass/requirements.txt
+
+COPY ${ctx_base} /opt/spyglass
+RUN pip3 install -e /opt/spyglass

requirements.txt

@@ -0,0 +1,7 @@
+jinja2==2.10
+jsonschema
+netaddr
+openpyxl==2.5.4
+pyyaml==3.12
+requests
+six

setup.py

@@ -0,0 +1,45 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from setuptools import setup
+from setuptools import find_packages
+
+setup(
+    name='spyglass',
+    version='0.0.1',
+    description='Generate Airship specific yaml manifests from data sources',
+    url='http://github.com/att-comdev/tugboat',
+    python_requires='>=3.5.0',
+    license='Apache 2.0',
+    packages=find_packages(),
+    install_requires=[
+        'jsonschema',
+        'Click',
+        'openpyxl',
+        'netaddr',
+        'pyyaml',
+        'jinja2',
+        'flask',
+        'flask-bootstrap',
+    ],
+    entry_points={
+        'console_scripts': [
+            'spyglass=spyglass.spyglass:main',
+        ],
+        'data_extractor_plugins':
+        ['formation=spyglass.data_extractor.plugins.formation:FormationPlugin',
+        ]
+    },
+    include_package_data=True,
+)

spyglass/data_extractor/base.py

@@ -0,0 +1,450 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+import pprint
+import six
+import logging
+
+from spyglass.utils import utils
+
+LOG = logging.getLogger(__name__)
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseDataSourcePlugin(object):
+    """Provide basic hooks for data source plugins"""
+
+    def __init__(self, region):
+        self.source_type = None
+        self.source_name = None
+        self.region = region
+        self.site_data = {}
+
+    @abc.abstractmethod
+    def set_config_opts(self, conf):
+        """Placeholder to set confgiuration options
+        specific to each plugin.
+
+        :param dict conf: Configuration options as dict
+
+        Example: conf = { 'excel_spec': 'spec1.yaml',
+                          'excel_path': 'excel.xls' }
+
+        Each plugin will have their own config opts.
+        """
+        return
+
+    @abc.abstractmethod
+    def get_plugin_conf(self, kwargs):
+        """ Validate and returns the plugin config parameters.
+        If validation fails, Spyglass exits.
+
+        :param char pointer: Spyglass CLI parameters.
+
+        :returns plugin conf if successfully validated.
+
+        Each plugin implements their own validaton mechanism.
+        """
+        return {}
+
+    @abc.abstractmethod
+    def get_racks(self, region):
+        """Return list of racks in the region
+
+        :param string region: Region name
+
+        :returns: list of rack names
+
+        :rtype: list
+
+        Example: ['rack01', 'rack02']
+        """
+        return []
+
+    @abc.abstractmethod
+    def get_hosts(self, region, rack):
+        """Return list of hosts in the region
+
+        :param string region: Region name
+        :param string rack: Rack name
+
+        :returns: list of hosts information
+
+        :rtype: list of dict
+
+        Example: [
+                     {
+                         'name': 'host01',
+                         'type': 'controller',
+                         'host_profile': 'hp_01'
+                     },
+                     {
+                         'name': 'host02',
+                         'type': 'compute',
+                         'host_profile': 'hp_02'}
+                 ]
+        """
+        return []
+
+    @abc.abstractmethod
+    def get_networks(self, region):
+        """Return list of networks in the region
+
+        :param string region: Region name
+
+        :returns: list of networks and their vlans
+
+        :rtype: list of dict
+
+        Example: [
+                     {
+                         'name': 'oob',
+                         'vlan': '41',
+                         'subnet': '192.168.1.0/24',
+                         'gateway': '192.168.1.1'
+                     },
+                     {
+                         'name': 'pxe',
+                         'vlan': '42',
+                         'subnet': '192.168.2.0/24',
+                         'gateway': '192.168.2.1'
+                     },
+                     {
+                         'name': 'oam',
+                         'vlan': '43',
+                         'subnet': '192.168.3.0/24',
+                         'gateway': '192.168.3.1'
+                     },
+                     {
+                         'name': 'ksn',
+                         'vlan': '44',
+                         'subnet': '192.168.4.0/24',
+                         'gateway': '192.168.4.1'
+                     },
+                     {
+                         'name': 'storage',
+                         'vlan': '45',
+                         'subnet': '192.168.5.0/24',
+                         'gateway': '192.168.5.1'
+                     },
+                     {
+                         'name': 'overlay',
+                         'vlan': '45',
+                         'subnet': '192.168.6.0/24',
+                         'gateway': '192.168.6.1'
+                     }
+                 ]
+        """
+
+        # TODO(nh863p): Expand the return type if they are rack level subnets
+        # TODO(nh863p): Is ingress information can be provided here?
+        return []
+
+    @abc.abstractmethod
+    def get_ips(self, region, host):
+        """Return list of IPs on the host
+
+        :param string region: Region name
+        :param string host: Host name
+
+        :returns: Dict of IPs per network on the host
+
+        :rtype: dict
+
+        Example: {'oob': {'ipv4': '192.168.1.10'},
+                  'pxe': {'ipv4': '192.168.2.10'}}
+
+        The network name from get_networks is expected to be the keys of this
+        dict. In case some networks are missed, they are expected to be either
+        DHCP or internally generated n the next steps by the design rules.
+        """
+        return {}
+
+    @abc.abstractmethod
+    def get_dns_servers(self, region):
+        """Return the DNS servers
+
+        :param string region: Region name
+
+        :returns: List of DNS servers to be configured on host
+
+        :rtype: List
+
+        Example: ['8.8.8.8', '8.8.8.4']
+        """
+        return []
+
+    @abc.abstractmethod
+    def get_ntp_servers(self, region):
+        """Return the NTP servers
+
+        :param string region: Region name
+
+        :returns: List of NTP servers to be configured on host
+
+        :rtype: List
+
+        Example: ['ntp1.ubuntu1.example', 'ntp2.ubuntu.example']
+        """
+        return []
+
+    @abc.abstractmethod
+    def get_ldap_information(self, region):
+        """Return the LDAP server information
+
+        :param string region: Region name
+
+        :returns: LDAP server information
+
+        :rtype: Dict
+
+        Example: {'url': 'ldap.example.com',
+                  'common_name': 'ldap-site1',
+                  'domain': 'test',
+                  'subdomain': 'test_sub1'}
+        """
+        return {}
+
+    @abc.abstractmethod
+    def get_location_information(self, region):
+        """Return location information
+
+        :param string region: Region name
+
+        :returns: Dict of location information
+
+        :rtype: dict
+
+        Example: {'name': 'Dallas',
+                  'physical_location': 'DAL01',
+                  'state': 'Texas',
+                  'country': 'US',
+                  'corridor': 'CR1'}
+        """
+        return {}
+
+    @abc.abstractmethod
+    def get_domain_name(self, region):
+        """Return the Domain name
+
+        :param string region: Region name
+
+        :returns: Domain name
+
+        :rtype: string
+
+        Example: example.com
+        """
+        return ""
+
+    def extract_baremetal_information(self):
+        """Get baremetal information from plugin
+
+        :returns: dict of baremetal nodes
+
+        :rtype: dict
+
+        Return dict should be in the format
+        {
+          'EXAMR06': {                 # rack name
+            'examr06c036': {           # host name
+              'host_profile': None,
+              'ip': {
+                'overlay': {},
+                'oob': {},
+                'calico': {},
+                'oam': {},
+                'storage': {},
+                'pxe': {}
+              },
+              'rack': 'EXAMR06',
+              'type': 'compute'
+            }
+          }
+        }
+        """
+        LOG.info("Extract baremetal information from plugin")
+        baremetal = {}
+        is_genesis = False
+        hosts = self.get_hosts(self.region)
+
+        # For each host list fill host profile and network IPs
+        for host in hosts:
+            host_name = host['name']
+            rack_name = host['rack_name']
+
+            if rack_name not in baremetal:
+                baremetal[rack_name] = {}
+
+            # Prepare temp dict for each host and append it to baremetal
+            # at a rack level
+            temp_host = {}
+            if host['host_profile'] is None:
+                temp_host['host_profile'] = "#CHANGE_ME"
+            else:
+                temp_host['host_profile'] = host['host_profile']
+
+            # Get Host IPs from plugin
+            temp_host_ips = self.get_ips(self.region, host_name)
+
+            # Fill network IP for this host
+            temp_host['ip'] = {}
+            temp_host['ip']['oob'] = temp_host_ips[host_name].get('oob', "")
+            temp_host['ip']['calico'] = temp_host_ips[host_name].get(
+                'calico', "")
+            temp_host['ip']['oam'] = temp_host_ips[host_name].get('oam', "")
+            temp_host['ip']['storage'] = temp_host_ips[host_name].get(
+                'storage', "")
+            temp_host['ip']['overlay'] = temp_host_ips[host_name].get(
+                'overlay', "")
+            temp_host['ip']['pxe'] = temp_host_ips[host_name].get(
+                'pxe', "#CHANGE_ME")
+
+            # Filling rack_type( compute/controller/genesis)
+            # "cp" host profile is controller
+            # "ns" host profile is compute
+            if (temp_host['host_profile'] == 'cp'):
+                # The controller node is designates as genesis"
+                if is_genesis is False:
+                    is_genesis = True
+                    temp_host['type'] = 'genesis'
+                else:
+                    temp_host['type'] = 'controller'
+            else:
+                temp_host['type'] = 'compute'
+
+            baremetal[rack_name][host_name] = temp_host
+        LOG.debug("Baremetal information:\n{}".format(
+            pprint.pformat(baremetal)))
+
+        return baremetal
+
+    def extract_site_information(self):
+        """Get site information from plugin
+
+        :returns: dict of site information
+
+        :rtpe: dict
+
+        Return dict should be in the format
+        {
+          'name': '',
+          'country': '',
+          'state': '',
+          'corridor': '',
+          'sitetype': '',
+          'dns': [],
+          'ntp': [],
+          'ldap': {},
+          'domain': None
+        }
+        """
+        LOG.info("Extract site information from plugin")
+        site_info = {}
+
+        # Extract location information
+        location_data = self.get_location_information(self.region)
+        if location_data is not None:
+            site_info = location_data
+
+        dns_data = self.get_dns_servers(self.region)
+        site_info['dns'] = dns_data
+
+        ntp_data = self.get_ntp_servers(self.region)
+        site_info['ntp'] = ntp_data
+
+        ldap_data = self.get_ldap_information(self.region)
+        site_info['ldap'] = ldap_data
+
+        domain_data = self.get_domain_name(self.region)
+        site_info['domain'] = domain_data
+
+        LOG.debug("Extracted site information:\n{}".format(
+            pprint.pformat(site_info)))
+
+        return site_info
+
+    def extract_network_information(self):
+        """Get network information from plugin
+        like Subnets, DNS, NTP, LDAP details.
+
+        :returns: dict of baremetal nodes
+
+        :rtype: dict
+
+        Return dict should be in the format
+        {
+          'vlan_network_data': {
+            'oam': {},
+            'ingress': {},
+            'oob': {}
+            'calico': {},
+            'storage': {},
+            'pxe': {},
+            'overlay': {}
+          }
+        }
+        """
+        LOG.info("Extract network information from plugin")
+        network_data = {}
+        networks = self.get_networks(self.region)
+
+        # We are interested in only the below networks mentioned in
+        # networks_to_scan, so look for these networks from the data
+        # returned by plugin
+        networks_to_scan = [
+            'calico', 'overlay', 'pxe', 'storage', 'oam', 'oob', 'ingress'
+        ]
+        network_data['vlan_network_data'] = {}
+
+        for net in networks:
+            tmp_net = {}
+            if net['name'] in networks_to_scan:
+                tmp_net['subnet'] = net['subnet']
+                tmp_net['vlan'] = net['vlan']
+
+            network_data['vlan_network_data'][net['name']] = tmp_net
+
+        LOG.debug("Extracted network data:\n{}".format(
+            pprint.pformat(network_data)))
+        return network_data
+
+    def extract_data(self):
+        """Extract data from plugin
+
+        Gather data related to baremetal, networks, storage and other site
+        related information from plugin
+        """
+        LOG.info("Extract data from plugin")
+        site_data = {}
+        site_data['baremetal'] = self.extract_baremetal_information()
+        site_data['site_info'] = self.extract_site_information()
+        site_data['network'] = self.extract_network_information()
+        self.site_data = site_data
+        return site_data
+
+    def apply_additional_data(self, extra_data):
+        """Apply any additional inputs from user
+
+        In case plugin doesnot provide some data, user can specify
+        the same as part of additional data in form of dict. The user
+        provided dict will be merged recursively to site_data.
+        If there is repetition of data then additional data supplied
+        shall take precedence.
+        """
+        LOG.info("Update site data with additional input")
+        tmp_site_data = utils.dict_merge(self.site_data, extra_data)
+        self.site_data = tmp_site_data
+        return self.site_data

spyglass/data_extractor/custom_exceptions.py

@@ -0,0 +1,46 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import sys
+
+LOG = logging.getLogger(__name__)
+
+
+class BaseError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+
+    def display_error(self):
+        LOG.info(self.msg)
+        sys.exit(1)
+
+
+class MissingAttributeError(BaseError):
+    pass
+
+
+class MissingValueError(BaseError):
+    pass
+
+
+class ApiClientError(BaseError):
+    pass
+
+
+class TokenGenerationError(BaseError):
+    pass
+
+
+class ConnectionError(BaseError):
+    pass

spyglass/data_extractor/plugins/formation.py

@@ -0,0 +1,496 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import pprint
+import re
+import requests
+import formation_client
+import urllib3
+
+from spyglass.data_extractor.base import BaseDataSourcePlugin
+
+from spyglass.data_extractor.custom_exceptions import (
+    ApiClientError, ConnectionError, MissingAttributeError,
+    TokenGenerationError)
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+LOG = logging.getLogger(__name__)
+
+
+class FormationPlugin(BaseDataSourcePlugin):
+    def __init__(self, region):
+        # Save site name is valid
+        try:
+            assert region is not None
+            super().__init__(region)
+        except AssertionError:
+            LOG.error("Site: None! Spyglass exited!")
+            LOG.info("Check spyglass --help for details")
+            exit()
+
+        self.source_type = 'rest'
+        self.source_name = 'formation'
+
+        # Configuration parameters
+        self.formation_api_url = None
+        self.user = None
+        self.password = None
+        self.token = None
+
+        # Formation objects
+        self.client_config = None
+        self.formation_api_client = None
+
+        # Site related data
+        self.region_zone_map = {}
+        self.site_name_id_mapping = {}
+        self.zone_name_id_mapping = {}
+        self.region_name_id_mapping = {}
+        self.rack_name_id_mapping = {}
+        self.device_name_id_mapping = {}
+        LOG.info("Initiated data extractor plugin:{}".format(self.source_name))
+
+    def set_config_opts(self, conf):
+        """ Sets the config params passed by CLI"""
+        LOG.info("Plugin params passed:\n{}".format(pprint.pformat(conf)))
+        self._validate_config_options(conf)
+        self.formation_api_url = conf['url']
+        self.user = conf['user']
+        self.password = conf['password']
+        self.token = conf.get('token', None)
+
+        self._get_formation_client()
+        self._update_site_and_zone(self.region)
+
+    def get_plugin_conf(self, kwargs):
+        """ Validates the plugin param and return if success"""
+        try:
+            assert (kwargs['formation_url']
+                    ) is not None, "formation_url is Not Specified"
+            url = kwargs['formation_url']
+            assert (kwargs['formation_user']
+                    ) is not None, "formation_user is Not Specified"
+            user = kwargs['formation_user']
+            assert (kwargs['formation_password']
+                    ) is not None, "formation_password is Not Specified"
+            password = kwargs['formation_password']
+        except AssertionError:
+            LOG.error("Insufficient plugin parameter! Spyglass exited!")
+            raise
+            exit()
+
+        plugin_conf = {'url': url, 'user': user, 'password': password}
+        return plugin_conf
+
+    def _validate_config_options(self, conf):
+        """Validate the CLI params passed
+
+        The method checks for missing parameters and terminates
+        Spyglass execution if found so.
+        """
+
+        missing_params = []
+        for key in conf.keys():
+            if conf[key] is None:
+                missing_params.append(key)
+        if len(missing_params) != 0:
+            LOG.error("Missing Plugin Params{}:".format(missing_params))
+            exit()
+
+    # Implement helper classes
+
+    def _generate_token(self):
+        """Generate token for Formation
+        Formation API does not provide separate resource to generate
+        token. This is a workaround to call directly Formation API
+        to get token instead of using Formation client.
+        """
+        # Create formation client config object
+        self.client_config = formation_client.Configuration()
+        self.client_config.host = self.formation_api_url
+        self.client_config.username = self.user
+        self.client_config.password = self.password
+        self.client_config.verify_ssl = False
+
+        # Assumes token is never expired in the execution of this tool
+        if self.token:
+            return self.token
+
+        url = self.formation_api_url + '/zones'
+        try:
+            token_response = requests.get(
+                url,
+                auth=(self.user, self.password),
+                verify=self.client_config.verify_ssl)
+        except requests.exceptions.ConnectionError:
+            raise ConnectionError('Incorrect URL: {}'.format(url))
+
+        if token_response.status_code == 200:
+            self.token = token_response.json().get('X-Subject-Token', None)
+        else:
+            raise TokenGenerationError(
+                'Unable to generate token because {}'.format(
+                    token_response.reason))
+
+        return self.token
+
+    def _get_formation_client(self):
+        """Create formation client object
+
+        Formation uses X-Auth-Token for authentication and should be in
+        format "user|token".
+        Generate the token and add it formation config object.
+        """
+        token = self._generate_token()
+        self.client_config.api_key = {'X-Auth-Token': self.user + '|' + token}
+        self.formation_api_client = formation_client.ApiClient(
+            self.client_config)
+
+    def _update_site_and_zone(self, region):
+        """Get Zone name and Site name from region"""
+
+        zone = self._get_zone_by_region_name(region)
+        site = self._get_site_by_zone_name(zone)
+
+        # zone = region[:-1]
+        # site = zone[:-1]
+
+        self.region_zone_map[region] = {}
+        self.region_zone_map[region]['zone'] = zone
+        self.region_zone_map[region]['site'] = site
+
+    def _get_zone_by_region_name(self, region_name):
+        zone_api = formation_client.ZonesApi(self.formation_api_client)
+        zones = zone_api.zones_get()
+
+        # Walk through each zone and get regions
+        # Return when region name matches
+        for zone in zones:
+            self.zone_name_id_mapping[zone.name] = zone.id
+            zone_regions = self.get_regions(zone.name)
+            if region_name in zone_regions:
+                return zone.name
+
+        return None
+
+    def _get_site_by_zone_name(self, zone_name):
+        site_api = formation_client.SitesApi(self.formation_api_client)
+        sites = site_api.sites_get()
+
+        # Walk through each site and get zones
+        # Return when site name matches
+        for site in sites:
+            self.site_name_id_mapping[site.name] = site.id
+            site_zones = self.get_zones(site.name)
+            if zone_name in site_zones:
+                return site.name
+
+        return None
+
+    def _get_site_id_by_name(self, site_name):
+        if site_name in self.site_name_id_mapping:
+            return self.site_name_id_mapping.get(site_name)
+
+        site_api = formation_client.SitesApi(self.formation_api_client)
+        sites = site_api.sites_get()
+        for site in sites:
+            self.site_name_id_mapping[site.name] = site.id
+            if site.name == site_name:
+                return site.id
+
+    def _get_zone_id_by_name(self, zone_name):
+        if zone_name in self.zone_name_id_mapping:
+            return self.zone_name_id_mapping.get(zone_name)
+
+        zone_api = formation_client.ZonesApi(self.formation_api_client)
+        zones = zone_api.zones_get()
+        for zone in zones:
+            if zone.name == zone_name:
+                self.zone_name_id_mapping[zone.name] = zone.id
+                return zone.id
+
+    def _get_region_id_by_name(self, region_name):
+        if region_name in self.region_name_id_mapping:
+            return self.region_name_id_mapping.get(region_name)
+
+        for zone in self.zone_name_id_mapping:
+            self.get_regions(zone)
+
+        return self.region_name_id_mapping.get(region_name, None)
+
+    def _get_rack_id_by_name(self, rack_name):
+        if rack_name in self.rack_name_id_mapping:
+            return self.rack_name_id_mapping.get(rack_name)
+
+        for zone in self.zone_name_id_mapping:
+            self.get_racks(zone)
+
+        return self.rack_name_id_mapping.get(rack_name, None)
+
+    def _get_device_id_by_name(self, device_name):
+        if device_name in self.device_name_id_mapping:
+            return self.device_name_id_mapping.get(device_name)
+
+        self.get_hosts(self.zone)
+
+        return self.device_name_id_mapping.get(device_name, None)
+
+    def _get_racks(self, zone, rack_type='compute'):
+        zone_id = self._get_zone_id_by_name(zone)
+        rack_api = formation_client.RacksApi(self.formation_api_client)
+        racks = rack_api.zones_zone_id_racks_get(zone_id)
+
+        racks_list = []
+        for rack in racks:
+            rack_name = rack.name
+            self.rack_name_id_mapping[rack_name] = rack.id
+            if rack.rack_type.name == rack_type:
+                racks_list.append(rack_name)
+
+        return racks_list
+
+    # Functions that will be used internally within this plugin
+
+    def get_zones(self, site=None):
+        zone_api = formation_client.ZonesApi(self.formation_api_client)
+
+        if site is None:
+            zones = zone_api.zones_get()
+        else:
+            site_id = self._get_site_id_by_name(site)
+            zones = zone_api.sites_site_id_zones_get(site_id)
+
+        zones_list = []
+        for zone in zones:
+            zone_name = zone.name
+            self.zone_name_id_mapping[zone_name] = zone.id
+            zones_list.append(zone_name)
+
+        return zones_list
+
+    def get_regions(self, zone):
+        zone_id = self._get_zone_id_by_name(zone)
+        region_api = formation_client.RegionApi(self.formation_api_client)
+        regions = region_api.zones_zone_id_regions_get(zone_id)
+        regions_list = []
+        for region in regions:
+            region_name = region.name
+            self.region_name_id_mapping[region_name] = region.id
+            regions_list.append(region_name)
+
+        return regions_list
+
+    # Implement Abstract functions
+
+    def get_racks(self, region):
+        zone = self.region_zone_map[region]['zone']
+        return self._get_racks(zone, rack_type='compute')
+
+    def get_hosts(self, region, rack=None):
+        zone = self.region_zone_map[region]['zone']
+        zone_id = self._get_zone_id_by_name(zone)
+        device_api = formation_client.DevicesApi(self.formation_api_client)
+        control_hosts = device_api.zones_zone_id_control_nodes_get(zone_id)
+        compute_hosts = device_api.zones_zone_id_devices_get(
+            zone_id, type='KVM')
+
+        hosts_list = []
+        for host in control_hosts:
+            self.device_name_id_mapping[host.aic_standard_name] = host.id
+            hosts_list.append({
+                'name': host.aic_standard_name,
+                'type': 'controller',
+                'rack_name': host.rack_name,
+                'host_profile': host.host_profile_name
+            })
+
+        for host in compute_hosts:
+            self.device_name_id_mapping[host.aic_standard_name] = host.id
+            hosts_list.append({
+                'name': host.aic_standard_name,
+                'type': 'compute',
+                'rack_name': host.rack_name,
+                'host_profile': host.host_profile_name
+            })
+        """
+        for host in itertools.chain(control_hosts, compute_hosts):
+            self.device_name_id_mapping[host.aic_standard_name] = host.id
+            hosts_list.append({
+            'name': host.aic_standard_name,
+            'type': host.categories[0],
+            'rack_name': host.rack_name,
+            'host_profile': host.host_profile_name
+            })
+        """
+
+        return hosts_list
+
+    def get_networks(self, region):
+        zone = self.region_zone_map[region]['zone']
+        zone_id = self._get_zone_id_by_name(zone)
+        region_id = self._get_region_id_by_name(region)
+        vlan_api = formation_client.VlansApi(self.formation_api_client)
+        vlans = vlan_api.zones_zone_id_regions_region_id_vlans_get(
+            zone_id, region_id)
+
+        # Case when vlans list is empty from
+        # zones_zone_id_regions_region_id_vlans_get
+        if len(vlans) is 0:
+            # get device-id from the first host and get the network details
+            hosts = self.get_hosts(self.region)
+            host = hosts[0]['name']
+            device_id = self._get_device_id_by_name(host)
+            vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get(
+                zone_id, device_id)
+
+        LOG.debug("Extracted region network information\n{}".format(vlans))
+        vlans_list = []
+        for vlan_ in vlans:
+            if len(vlan_.vlan.ipv4) is not 0:
+                tmp_vlan = {}
+                tmp_vlan['name'] = self._get_network_name_from_vlan_name(
+                    vlan_.vlan.name)
+                tmp_vlan['vlan'] = vlan_.vlan.vlan_id
+                tmp_vlan['subnet'] = vlan_.vlan.subnet_range
+                tmp_vlan['gateway'] = vlan_.ipv4_gateway
+                tmp_vlan['subnet_level'] = vlan_.vlan.subnet_level
+                vlans_list.append(tmp_vlan)
+
+        return vlans_list
+
+    def get_ips(self, region, host=None):
+        zone = self.region_zone_map[region]['zone']
+        zone_id = self._get_zone_id_by_name(zone)
+
+        if host:
+            hosts = [host]
+        else:
+            hosts = []
+            hosts_dict = self.get_hosts(zone)
+            for host in hosts_dict:
+                hosts.append(host['name'])
+
+        vlan_api = formation_client.VlansApi(self.formation_api_client)
+        ip_ = {}
+
+        for host in hosts:
+            device_id = self._get_device_id_by_name(host)
+            vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get(
+                zone_id, device_id)
+            LOG.debug("Received VLAN Network Information\n{}".format(vlans))
+            ip_[host] = {}
+            for vlan_ in vlans:
+                # TODO(pg710r) We need to handle the case when incoming ipv4
+                # list is empty
+                if len(vlan_.vlan.ipv4) is not 0:
+                    name = self._get_network_name_from_vlan_name(
+                        vlan_.vlan.name)
+                    ipv4 = vlan_.vlan.ipv4[0].ip
+                    LOG.debug("vlan:{},name:{},ip:{},vlan_name:{}".format(
+                        vlan_.vlan.vlan_id, name, ipv4, vlan_.vlan.name))
+                    # TODD(pg710r) This code needs to extended to support ipv4
+                    # and ipv6
+                    # ip_[host][name] = {'ipv4': ipv4}
+                    ip_[host][name] = ipv4
+
+        return ip_
+
+    def _get_network_name_from_vlan_name(self, vlan_name):
+        """ network names are ksn, oam, oob, overlay, storage, pxe
+
+        The following mapping rules apply:
+            vlan_name contains "ksn"  the network name is "calico"
+            vlan_name contains "storage" the network name is "storage"
+            vlan_name contains "server"  the network name is "oam"
+            vlan_name contains "ovs"  the network name is "overlay"
+            vlan_name contains "ILO" the network name is "oob"
+        """
+        network_names = {
+            'ksn': 'calico',
+            'storage': 'storage',
+            'server': 'oam',
+            'ovs': 'overlay',
+            'ILO': 'oob',
+            'pxe': 'pxe'
+        }
+
+        for name in network_names:
+            # Make a pattern that would ignore case.
+            # if name is 'ksn' pattern name is '(?i)(ksn)'
+            name_pattern = "(?i)({})".format(name)
+            if re.search(name_pattern, vlan_name):
+                return network_names[name]
+
+        return ("")
+
+    def get_dns_servers(self, region):
+        try:
+            zone = self.region_zone_map[region]['zone']
+            zone_id = self._get_zone_id_by_name(zone)
+            zone_api = formation_client.ZonesApi(self.formation_api_client)
+            zone_ = zone_api.zones_zone_id_get(zone_id)
+        except formation_client.rest.ApiException as e:
+            raise ApiClientError(e.msg)
+
+        if not zone_.ipv4_dns:
+            LOG.warn("No dns server")
+            return []
+
+        dns_list = []
+        for dns in zone_.ipv4_dns:
+            dns_list.append(dns.ip)
+
+        return dns_list
+
+    def get_ntp_servers(self, region):
+        return []
+
+    def get_ldap_information(self, region):
+        return {}
+
+    def get_location_information(self, region):
+        """ get location information for a zone and return """