airship
/
spyglass
Code Issues Proposed changes

Refactors CLI to match style of Airship CLIs 

The original CLI is based on click, but includes all options stacked on
one main method. This is in contrast to other Airship CLIs such as
Pegleg which utilizes click's nesting features to organize its CLI into
multiple groups and commands that each have their own options.

This change separates the Spyglass CLI into three different commands:
generate intermediary, generate manifests, and generate manifests from
intermediary.

Adds a 'verbose' flag on Spyglass.

Defaults plugin to 'tugboat'.

Adds validation for options that apply specifically to tugboat or
formation.

Related docs change: https://review.opendev.org/#/c/650137/

Change-Id: I92e5f040d5205c3ab36ec1d46ecd57bc97849cef
This commit is contained in:
Ian Pittwood 2019-04-02 15:47:49 -05:00
parent 69f0975dec
commit c46e15385c
5 changed files with 287 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
requirements.txt
View File

@ -3,4 +3,5 @@ jsonschema
netaddr
openpyxl==2.5.4
pyyaml==3.12
requests
requests
click

2
setup.py
View File

@ -33,7 +33,7 @@ setup(
],
entry_points={
'console_scripts': [
'spyglass=spyglass.spyglass:main',
'spyglass=spyglass.cli:main',
],
'data_extractor_plugins': [
'formation='

283
spyglass/cli.py Normal file
View File

@ -0,0 +1,283 @@
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import pprint
import click
import pkg_resources
import yaml
from spyglass.parser.engine import ProcessDataSource
from spyglass.site_processors.site_processor import SiteProcessor
LOG = logging.getLogger(__name__)
LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:' \
'%(funcName)s [%(lineno)3d] %(message)s'
CONTEXT_SETTINGS = {
'help_option_names': ['-h', '--help'],
}
def tugboat_required_callback(ctx, param, value):
LOG.debug('Evaluating %s: %s', param.name, value)
if 'plugin_type' not in ctx.params or \
ctx.params['plugin_type'] == 'tugboat':
if not value:
raise click.UsageError('%s is required for the tugboat '
'plugin.' % str(param.name),
ctx=ctx)
return value
def formation_required_callback(ctx, param, value):
LOG.debug('Evaluating %s: %s', param.name, value)
if 'plugin_type' in ctx.params:
if ctx.params['plugin_type'] == 'formation':
if not value:
raise click.UsageError('%s is required for the '
'formation plugin.' % str(param.name),
ctx=ctx)
return value
return ['', '', '']
PLUGIN_TYPE_OPTION = click.option(
'-p',
'--plugin-type',
'plugin_type',
type=click.Choice(['formation', 'tugboat']),
default='tugboat',
show_default=True,
help='The plugin type to use.') # yapf: disable
# TODO(ianp): Either provide a prompt for passwords or use environment
# variable so passwords are no longer plain text
FORMATION_TARGET_OPTION = click.option(
'-f',
'--formation-target',
'formation_target',
nargs=3,
help=('Target URL, username, and password for formation plugin. Required '
'for formation plugin.'),
callback=formation_required_callback)
INTERMEDIARY_DIR_OPTION = click.option(
'-d',
'--intermediary-dir',
'intermediary_dir',
type=click.Path(exists=True, file_okay=False, writable=True),
default='./',
help='Directory in which the intermediary file will be created.')
EXCEL_FILE_OPTION = click.option(
'-x',
'--excel-file',
'excel_file',
multiple=True,
type=click.Path(exists=True, readable=True, dir_okay=False),
help='Path to the engineering Excel file. Required for tugboat plugin.',
callback=tugboat_required_callback)
EXCEL_SPEC_OPTION = click.option(
'-e',
'--excel-spec',
'excel_spec',
type=click.Path(exists=True, readable=True, dir_okay=False),
help=('Path to the Excel specification YAML file for the engineering '
'Excel file. Required for tugboat plugin.'),
callback=tugboat_required_callback)
SITE_CONFIGURATION_FILE_OPTION = click.option(
'-c',
'--site-configuration',
'site_configuration',
type=click.Path(exists=True, readable=True, dir_okay=False),
required=False,
help='Path to site specific configuration details YAML file.')
SITE_NAME_CONFIGURATION_OPTION = click.option(
'-s',
'--site-name',
'site_name',
type=click.STRING,
required=False,
help='Name of the site for which the intermediary is being generated.')
TEMPLATE_DIR_OPTION = click.option(
'-t',
'--template-dir',
'template_dir',
type=click.Path(exists=True, readable=True, file_okay=False),
required=True,
help='Path to the directory containing manifest J2 templates.')
MANIFEST_DIR_OPTION = click.option(
'-m',
'--manifest-dir',
'manifest_dir',
type=click.Path(exists=True, writable=True, file_okay=False),
required=False,
help='Path to place created manifest files.')
@click.group(context_settings=CONTEXT_SETTINGS)
@click.option('-v',
'--verbose',
is_flag=True,
default=False,
help='Enable debug messages in log.')
def main(*, verbose):
"""CLI for Airship Spyglass"""
if verbose:
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.basicConfig(format=LOG_FORMAT, level=log_level)
def _intermediary_helper(plugin_type, formation_data, site, excel_file,
excel_spec, additional_configuration):
LOG.info("Generating Intermediary yaml")
plugin_type = plugin_type
plugin_class = None
# Discover the plugin and load the plugin class
LOG.info("Load the plugin class")
for entry_point in \
pkg_resources.iter_entry_points('data_extractor_plugins'):
if entry_point.name == plugin_type:
plugin_class = entry_point.load()
if plugin_class is None:
LOG.error(
"Unsupported Plugin type. Plugin type:{}".format(plugin_type))
exit()
# Extract data from plugin data source
LOG.info("Extract data from plugin data source")
data_extractor = plugin_class(site)
plugin_conf = data_extractor.get_plugin_conf({
'excel': excel_file,
'excel_spec': excel_spec,
'formation_url': formation_data[0],
'formation_user': formation_data[1],
'formation_password': formation_data[2]
}) # yapf: disable
data_extractor.set_config_opts(plugin_conf)
data_extractor.extract_data()
# Apply any additional_config provided by user
additional_config = additional_configuration
if additional_config is not None:
with open(additional_config, 'r') as config:
raw_data = config.read()
additional_config_data = yaml.safe_load(raw_data)
LOG.debug("Additional config data:\n{}".format(
pprint.pformat(additional_config_data)))
LOG.info(
"Apply additional configuration from:{}".format(additional_config))
data_extractor.apply_additional_data(additional_config_data)
LOG.debug(pprint.pformat(data_extractor.site_data))
# Apply design rules to the data
LOG.info("Apply design rules to the extracted data")
process_input_ob = ProcessDataSource(site)
process_input_ob.load_extracted_data_from_data_source(
data_extractor.site_data)
return process_input_ob
@main.command('i',
short_help='generate intermediary',
help='Generates an intermediary file from passed excel data.')
@PLUGIN_TYPE_OPTION
@FORMATION_TARGET_OPTION
@INTERMEDIARY_DIR_OPTION
@EXCEL_FILE_OPTION
@EXCEL_SPEC_OPTION
@SITE_CONFIGURATION_FILE_OPTION
@SITE_NAME_CONFIGURATION_OPTION
def generate_intermediary(*, plugin_type, formation_target, intermediary_dir,
excel_file, excel_spec, site_configuration,
site_name):
process_input_ob = _intermediary_helper(plugin_type, formation_target,
site_name, excel_file, excel_spec,
site_configuration)
LOG.info("Generate intermediary yaml")
process_input_ob.generate_intermediary_yaml()
process_input_ob.dump_intermediary_file(intermediary_dir)
@main.command('m',
short_help='generates manifest and intermediary',
help='Generates manifest and intermediary files.')
@click.option(
'-i',
'--save-intermediary',
'save_intermediary',
is_flag=True,
default=False,
help='Flag to save the generated intermediary file used for the manifests.'
)
@PLUGIN_TYPE_OPTION
@FORMATION_TARGET_OPTION
@INTERMEDIARY_DIR_OPTION
@EXCEL_FILE_OPTION
@EXCEL_SPEC_OPTION
@SITE_CONFIGURATION_FILE_OPTION
@SITE_NAME_CONFIGURATION_OPTION
@TEMPLATE_DIR_OPTION
@MANIFEST_DIR_OPTION
def generate_manifests_and_intermediary(*, save_intermediary, plugin_type,
formation_target, intermediary_dir,
excel_file, excel_spec,
site_configuration, site_name,
template_dir, manifest_dir):
process_input_ob = _intermediary_helper(plugin_type, formation_target,
site_name, excel_file, excel_spec,
site_configuration)
LOG.info("Generate intermediary yaml")
intermediary_yaml = process_input_ob.generate_intermediary_yaml()
if save_intermediary:
LOG.debug("Dumping intermediary yaml")
process_input_ob.dump_intermediary_file(intermediary_dir)
else:
LOG.debug("Skipping dump for intermediary yaml")
LOG.info("Generating site Manifests")
processor_engine = SiteProcessor(intermediary_yaml, manifest_dir)
processor_engine.render_template(template_dir)
@main.command('mi',
short_help='generates manifest from intermediary',
help='Generate manifest files from specified intermediary file.')
@click.argument('intermediary_file',
type=click.Path(exists=True, readable=True, dir_okay=False))
@TEMPLATE_DIR_OPTION
@MANIFEST_DIR_OPTION
def generate_manifests_using_intermediary(*, intermediary_file, template_dir,
manifest_dir):
LOG.info("Loading intermediary from user provided input")
with open(intermediary_file, 'r') as f:
raw_data = f.read()
intermediary_yaml = yaml.safe_load(raw_data)
LOG.info("Generating site Manifests")
processor_engine = SiteProcessor(intermediary_yaml, manifest_dir)
processor_engine.render_template(template_dir)

2
spyglass/parser/engine.py
View File

@ -310,7 +310,7 @@ class ProcessDataSource(object):
pprint.pformat(vlan_network_data_)))
def load_extracted_data_from_data_source(self, extracted_data):
"""Function called from spyglass.py to pass extracted data
"""Function called from cli.py to pass extracted data
from input data source
"""

206
spyglass/spyglass.py
View File

@ -1,206 +0,0 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import pprint
import click
from pkg_resources import iter_entry_points
import yaml
from spyglass.parser.engine import ProcessDataSource
from spyglass.site_processors.site_processor import SiteProcessor
LOG = logging.getLogger("spyglass")
@click.command()
@click.option("--site",
"-s",
help="Specify the site for which manifests to be generated")
@click.option("--type",
"-t",
help="Specify the plugin type formation or tugboat")
@click.option("--formation_url", "-f", help="Specify the formation url")
@click.option("--formation_user", "-u", help="Specify the formation user id")
@click.option("--formation_password",
"-p",
help="Specify the formation user password")
@click.option(
"--intermediary",
"-i",
type=click.Path(exists=True),
help="Intermediary file path generate manifests, "
"use -m also with this option",
)
@click.option(
"--additional_config",
"-d",
type=click.Path(exists=True),
help="Site specific configuraton details",
)
@click.option(
"--generate_intermediary",
"-g",
is_flag=True,
help="Dump intermediary file from passed excel and excel spec",
)
@click.option(
"--intermediary_dir",
"-idir",
type=click.Path(exists=True),
help="The path where intermediary file needs to be generated",
)
@click.option(
"--generate_manifests",
"-m",
is_flag=True,
help="Generate manifests from the generated intermediary file",
)
@click.option(
"--manifest_dir",
"-mdir",
type=click.Path(exists=True),
help="The path where manifest files needs to be generated",
)
@click.option(
"--template_dir",
"-tdir",
type=click.Path(exists=True),
help="The path where J2 templates are available",
)
@click.option(
"--excel",
"-x",
multiple=True,
type=click.Path(exists=True),
help="Path to engineering excel file, to be passed with "
"generate_intermediary",
)
@click.option(
"--excel_spec",
"-e",
type=click.Path(exists=True),
help="Path to excel spec, to be passed with generate_intermediary",
)
@click.option(
"--loglevel",
"-l",
default=20,
multiple=False,
show_default=True,
help="Loglevel NOTSET:0 ,DEBUG:10, INFO:20, WARNING:30, ERROR:40, "
"CRITICAL:50",
)
def main(*args, **kwargs):
# Extract user provided inputs
generate_intermediary = kwargs["generate_intermediary"]
intermediary_dir = kwargs["intermediary_dir"]
generate_manifests = kwargs["generate_manifests"]
manifest_dir = kwargs["manifest_dir"]
intermediary = kwargs["intermediary"]
site = kwargs["site"]
template_dir = kwargs["template_dir"]
loglevel = kwargs["loglevel"]
# Set Logging format
LOG.setLevel(loglevel)
stream_handle = logging.StreamHandler()
formatter = \
logging.Formatter("(%(name)s): %(asctime)s %(levelname)s %(message)s")
stream_handle.setFormatter(formatter)
LOG.addHandler(stream_handle)
LOG.info("Spyglass start")
LOG.info("CLI Parameters passed:\n{}".format(kwargs))
if not (generate_intermediary or generate_manifests):
LOG.error("Invalid CLI parameters passed!! Spyglass exited")
LOG.error("One of the options -m/-g is mandatory")
LOG.info("CLI Parameters:\n{}".format(kwargs))
exit()
if generate_manifests:
if template_dir is None:
LOG.error("Template directory not specified!! Spyglass exited")
LOG.error("It is mandatory to provide it when "
"generate_manifests is true")
exit()
# Generate Intermediary yaml and manifests extracting data
# from data source specified by plugin type
if intermediary is None:
LOG.info("Generating Intermediary yaml")
plugin_type = kwargs.get("type", None)
plugin_class = None
# Discover the plugin and load the plugin class
LOG.info("Load the plugin class")
for entry_point in iter_entry_points("data_extractor_plugins"):
if entry_point.name == plugin_type:
plugin_class = entry_point.load()
if plugin_class is None:
LOG.error(
"Unsupported Plugin type. Plugin type:{}".format(plugin_type))
exit()
# Extract data from plugin data source
LOG.info("Extract data from plugin data source")
data_extractor = plugin_class(site)
plugin_conf = data_extractor.get_plugin_conf(kwargs)
data_extractor.set_config_opts(plugin_conf)
data_extractor.extract_data()
# Apply any additional_config provided by user
additional_config = kwargs.get("additional_config", None)
if additional_config is not None:
with open(additional_config, "r") as config:
raw_data = config.read()
additional_config_data = yaml.safe_load(raw_data)
LOG.debug("Additional config data:\n{}".format(
pprint.pformat(additional_config_data)))
LOG.info("Apply additional configuration from:{}".format(
additional_config))
data_extractor.apply_additional_data(additional_config_data)
LOG.debug(pprint.pformat(data_extractor.site_data))
# Apply design rules to the data
LOG.info("Apply design rules to the extracted data")
process_input_ob = ProcessDataSource(site)
process_input_ob.load_extracted_data_from_data_source(
data_extractor.site_data)
LOG.info("Generate intermediary yaml")
intermediary_yaml = process_input_ob.generate_intermediary_yaml()
else:
LOG.info("Loading intermediary from user provided input")
with open(intermediary, "r") as intermediary_file:
raw_data = intermediary_file.read()
intermediary_yaml = yaml.safe_load(raw_data)
if generate_intermediary:
process_input_ob.dump_intermediary_file(intermediary_dir)
if generate_manifests:
LOG.info("Generating site Manifests")
processor_engine = SiteProcessor(intermediary_yaml, manifest_dir)
processor_engine.render_template(template_dir)
LOG.info("Spyglass Execution Completed")
if __name__ == "__main__":
main()