497 lines
18 KiB
Python
497 lines
18 KiB
Python
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the 'License');
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an 'AS IS' BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import logging
|
|
import pprint
|
|
import re
|
|
import requests
|
|
import formation_client
|
|
import urllib3
|
|
|
|
from spyglass.data_extractor.base import BaseDataSourcePlugin
|
|
|
|
from spyglass.data_extractor.custom_exceptions import (
|
|
ApiClientError, ConnectionError, MissingAttributeError,
|
|
TokenGenerationError)
|
|
|
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class FormationPlugin(BaseDataSourcePlugin):
|
|
def __init__(self, region):
|
|
# Save site name is valid
|
|
try:
|
|
assert region is not None
|
|
super().__init__(region)
|
|
except AssertionError:
|
|
LOG.error("Site: None! Spyglass exited!")
|
|
LOG.info("Check spyglass --help for details")
|
|
exit()
|
|
|
|
self.source_type = 'rest'
|
|
self.source_name = 'formation'
|
|
|
|
# Configuration parameters
|
|
self.formation_api_url = None
|
|
self.user = None
|
|
self.password = None
|
|
self.token = None
|
|
|
|
# Formation objects
|
|
self.client_config = None
|
|
self.formation_api_client = None
|
|
|
|
# Site related data
|
|
self.region_zone_map = {}
|
|
self.site_name_id_mapping = {}
|
|
self.zone_name_id_mapping = {}
|
|
self.region_name_id_mapping = {}
|
|
self.rack_name_id_mapping = {}
|
|
self.device_name_id_mapping = {}
|
|
LOG.info("Initiated data extractor plugin:{}".format(self.source_name))
|
|
|
|
def set_config_opts(self, conf):
|
|
""" Sets the config params passed by CLI"""
|
|
LOG.info("Plugin params passed:\n{}".format(pprint.pformat(conf)))
|
|
self._validate_config_options(conf)
|
|
self.formation_api_url = conf['url']
|
|
self.user = conf['user']
|
|
self.password = conf['password']
|
|
self.token = conf.get('token', None)
|
|
|
|
self._get_formation_client()
|
|
self._update_site_and_zone(self.region)
|
|
|
|
def get_plugin_conf(self, kwargs):
|
|
""" Validates the plugin param and return if success"""
|
|
try:
|
|
assert (kwargs['formation_url']
|
|
) is not None, "formation_url is Not Specified"
|
|
url = kwargs['formation_url']
|
|
assert (kwargs['formation_user']
|
|
) is not None, "formation_user is Not Specified"
|
|
user = kwargs['formation_user']
|
|
assert (kwargs['formation_password']
|
|
) is not None, "formation_password is Not Specified"
|
|
password = kwargs['formation_password']
|
|
except AssertionError:
|
|
LOG.error("Insufficient plugin parameter! Spyglass exited!")
|
|
raise
|
|
exit()
|
|
|
|
plugin_conf = {'url': url, 'user': user, 'password': password}
|
|
return plugin_conf
|
|
|
|
def _validate_config_options(self, conf):
|
|
"""Validate the CLI params passed
|
|
|
|
The method checks for missing parameters and terminates
|
|
Spyglass execution if found so.
|
|
"""
|
|
|
|
missing_params = []
|
|
for key in conf.keys():
|
|
if conf[key] is None:
|
|
missing_params.append(key)
|
|
if len(missing_params) != 0:
|
|
LOG.error("Missing Plugin Params{}:".format(missing_params))
|
|
exit()
|
|
|
|
# Implement helper classes
|
|
|
|
def _generate_token(self):
|
|
"""Generate token for Formation
|
|
Formation API does not provide separate resource to generate
|
|
token. This is a workaround to call directly Formation API
|
|
to get token instead of using Formation client.
|
|
"""
|
|
# Create formation client config object
|
|
self.client_config = formation_client.Configuration()
|
|
self.client_config.host = self.formation_api_url
|
|
self.client_config.username = self.user
|
|
self.client_config.password = self.password
|
|
self.client_config.verify_ssl = False
|
|
|
|
# Assumes token is never expired in the execution of this tool
|
|
if self.token:
|
|
return self.token
|
|
|
|
url = self.formation_api_url + '/zones'
|
|
try:
|
|
token_response = requests.get(
|
|
url,
|
|
auth=(self.user, self.password),
|
|
verify=self.client_config.verify_ssl)
|
|
except requests.exceptions.ConnectionError:
|
|
raise ConnectionError('Incorrect URL: {}'.format(url))
|
|
|
|
if token_response.status_code == 200:
|
|
self.token = token_response.json().get('X-Subject-Token', None)
|
|
else:
|
|
raise TokenGenerationError(
|
|
'Unable to generate token because {}'.format(
|
|
token_response.reason))
|
|
|
|
return self.token
|
|
|
|
def _get_formation_client(self):
|
|
"""Create formation client object
|
|
|
|
Formation uses X-Auth-Token for authentication and should be in
|
|
format "user|token".
|
|
Generate the token and add it formation config object.
|
|
"""
|
|
token = self._generate_token()
|
|
self.client_config.api_key = {'X-Auth-Token': self.user + '|' + token}
|
|
self.formation_api_client = formation_client.ApiClient(
|
|
self.client_config)
|
|
|
|
def _update_site_and_zone(self, region):
|
|
"""Get Zone name and Site name from region"""
|
|
|
|
zone = self._get_zone_by_region_name(region)
|
|
site = self._get_site_by_zone_name(zone)
|
|
|
|
# zone = region[:-1]
|
|
# site = zone[:-1]
|
|
|
|
self.region_zone_map[region] = {}
|
|
self.region_zone_map[region]['zone'] = zone
|
|
self.region_zone_map[region]['site'] = site
|
|
|
|
def _get_zone_by_region_name(self, region_name):
|
|
zone_api = formation_client.ZonesApi(self.formation_api_client)
|
|
zones = zone_api.zones_get()
|
|
|
|
# Walk through each zone and get regions
|
|
# Return when region name matches
|
|
for zone in zones:
|
|
self.zone_name_id_mapping[zone.name] = zone.id
|
|
zone_regions = self.get_regions(zone.name)
|
|
if region_name in zone_regions:
|
|
return zone.name
|
|
|
|
return None
|
|
|
|
def _get_site_by_zone_name(self, zone_name):
|
|
site_api = formation_client.SitesApi(self.formation_api_client)
|
|
sites = site_api.sites_get()
|
|
|
|
# Walk through each site and get zones
|
|
# Return when site name matches
|
|
for site in sites:
|
|
self.site_name_id_mapping[site.name] = site.id
|
|
site_zones = self.get_zones(site.name)
|
|
if zone_name in site_zones:
|
|
return site.name
|
|
|
|
return None
|
|
|
|
def _get_site_id_by_name(self, site_name):
|
|
if site_name in self.site_name_id_mapping:
|
|
return self.site_name_id_mapping.get(site_name)
|
|
|
|
site_api = formation_client.SitesApi(self.formation_api_client)
|
|
sites = site_api.sites_get()
|
|
for site in sites:
|
|
self.site_name_id_mapping[site.name] = site.id
|
|
if site.name == site_name:
|
|
return site.id
|
|
|
|
def _get_zone_id_by_name(self, zone_name):
|
|
if zone_name in self.zone_name_id_mapping:
|
|
return self.zone_name_id_mapping.get(zone_name)
|
|
|
|
zone_api = formation_client.ZonesApi(self.formation_api_client)
|
|
zones = zone_api.zones_get()
|
|
for zone in zones:
|
|
if zone.name == zone_name:
|
|
self.zone_name_id_mapping[zone.name] = zone.id
|
|
return zone.id
|
|
|
|
def _get_region_id_by_name(self, region_name):
|
|
if region_name in self.region_name_id_mapping:
|
|
return self.region_name_id_mapping.get(region_name)
|
|
|
|
for zone in self.zone_name_id_mapping:
|
|
self.get_regions(zone)
|
|
|
|
return self.region_name_id_mapping.get(region_name, None)
|
|
|
|
def _get_rack_id_by_name(self, rack_name):
|
|
if rack_name in self.rack_name_id_mapping:
|
|
return self.rack_name_id_mapping.get(rack_name)
|
|
|
|
for zone in self.zone_name_id_mapping:
|
|
self.get_racks(zone)
|
|
|
|
return self.rack_name_id_mapping.get(rack_name, None)
|
|
|
|
def _get_device_id_by_name(self, device_name):
|
|
if device_name in self.device_name_id_mapping:
|
|
return self.device_name_id_mapping.get(device_name)
|
|
|
|
self.get_hosts(self.zone)
|
|
|
|
return self.device_name_id_mapping.get(device_name, None)
|
|
|
|
def _get_racks(self, zone, rack_type='compute'):
|
|
zone_id = self._get_zone_id_by_name(zone)
|
|
rack_api = formation_client.RacksApi(self.formation_api_client)
|
|
racks = rack_api.zones_zone_id_racks_get(zone_id)
|
|
|
|
racks_list = []
|
|
for rack in racks:
|
|
rack_name = rack.name
|
|
self.rack_name_id_mapping[rack_name] = rack.id
|
|
if rack.rack_type.name == rack_type:
|
|
racks_list.append(rack_name)
|
|
|
|
return racks_list
|
|
|
|
# Functions that will be used internally within this plugin
|
|
|
|
def get_zones(self, site=None):
|
|
zone_api = formation_client.ZonesApi(self.formation_api_client)
|
|
|
|
if site is None:
|
|
zones = zone_api.zones_get()
|
|
else:
|
|
site_id = self._get_site_id_by_name(site)
|
|
zones = zone_api.sites_site_id_zones_get(site_id)
|
|
|
|
zones_list = []
|
|
for zone in zones:
|
|
zone_name = zone.name
|
|
self.zone_name_id_mapping[zone_name] = zone.id
|
|
zones_list.append(zone_name)
|
|
|
|
return zones_list
|
|
|
|
def get_regions(self, zone):
|
|
zone_id = self._get_zone_id_by_name(zone)
|
|
region_api = formation_client.RegionApi(self.formation_api_client)
|
|
regions = region_api.zones_zone_id_regions_get(zone_id)
|
|
regions_list = []
|
|
for region in regions:
|
|
region_name = region.name
|
|
self.region_name_id_mapping[region_name] = region.id
|
|
regions_list.append(region_name)
|
|
|
|
return regions_list
|
|
|
|
# Implement Abstract functions
|
|
|
|
def get_racks(self, region):
|
|
zone = self.region_zone_map[region]['zone']
|
|
return self._get_racks(zone, rack_type='compute')
|
|
|
|
def get_hosts(self, region, rack=None):
|
|
zone = self.region_zone_map[region]['zone']
|
|
zone_id = self._get_zone_id_by_name(zone)
|
|
device_api = formation_client.DevicesApi(self.formation_api_client)
|
|
control_hosts = device_api.zones_zone_id_control_nodes_get(zone_id)
|
|
compute_hosts = device_api.zones_zone_id_devices_get(
|
|
zone_id, type='KVM')
|
|
|
|
hosts_list = []
|
|
for host in control_hosts:
|
|
self.device_name_id_mapping[host.aic_standard_name] = host.id
|
|
hosts_list.append({
|
|
'name': host.aic_standard_name,
|
|
'type': 'controller',
|
|
'rack_name': host.rack_name,
|
|
'host_profile': host.host_profile_name
|
|
})
|
|
|
|
for host in compute_hosts:
|
|
self.device_name_id_mapping[host.aic_standard_name] = host.id
|
|
hosts_list.append({
|
|
'name': host.aic_standard_name,
|
|
'type': 'compute',
|
|
'rack_name': host.rack_name,
|
|
'host_profile': host.host_profile_name
|
|
})
|
|
"""
|
|
for host in itertools.chain(control_hosts, compute_hosts):
|
|
self.device_name_id_mapping[host.aic_standard_name] = host.id
|
|
hosts_list.append({
|
|
'name': host.aic_standard_name,
|
|
'type': host.categories[0],
|
|
'rack_name': host.rack_name,
|
|
'host_profile': host.host_profile_name
|
|
})
|
|
"""
|
|
|
|
return hosts_list
|
|
|
|
def get_networks(self, region):
|
|
zone = self.region_zone_map[region]['zone']
|
|
zone_id = self._get_zone_id_by_name(zone)
|
|
region_id = self._get_region_id_by_name(region)
|
|
vlan_api = formation_client.VlansApi(self.formation_api_client)
|
|
vlans = vlan_api.zones_zone_id_regions_region_id_vlans_get(
|
|
zone_id, region_id)
|
|
|
|
# Case when vlans list is empty from
|
|
# zones_zone_id_regions_region_id_vlans_get
|
|
if len(vlans) is 0:
|
|
# get device-id from the first host and get the network details
|
|
hosts = self.get_hosts(self.region)
|
|
host = hosts[0]['name']
|
|
device_id = self._get_device_id_by_name(host)
|
|
vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get(
|
|
zone_id, device_id)
|
|
|
|
LOG.debug("Extracted region network information\n{}".format(vlans))
|
|
vlans_list = []
|
|
for vlan_ in vlans:
|
|
if len(vlan_.vlan.ipv4) is not 0:
|
|
tmp_vlan = {}
|
|
tmp_vlan['name'] = self._get_network_name_from_vlan_name(
|
|
vlan_.vlan.name)
|
|
tmp_vlan['vlan'] = vlan_.vlan.vlan_id
|
|
tmp_vlan['subnet'] = vlan_.vlan.subnet_range
|
|
tmp_vlan['gateway'] = vlan_.ipv4_gateway
|
|
tmp_vlan['subnet_level'] = vlan_.vlan.subnet_level
|
|
vlans_list.append(tmp_vlan)
|
|
|
|
return vlans_list
|
|
|
|
def get_ips(self, region, host=None):
|
|
zone = self.region_zone_map[region]['zone']
|
|
zone_id = self._get_zone_id_by_name(zone)
|
|
|
|
if host:
|
|
hosts = [host]
|
|
else:
|
|
hosts = []
|
|
hosts_dict = self.get_hosts(zone)
|
|
for host in hosts_dict:
|
|
hosts.append(host['name'])
|
|
|
|
vlan_api = formation_client.VlansApi(self.formation_api_client)
|
|
ip_ = {}
|
|
|
|
for host in hosts:
|
|
device_id = self._get_device_id_by_name(host)
|
|
vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get(
|
|
zone_id, device_id)
|
|
LOG.debug("Received VLAN Network Information\n{}".format(vlans))
|
|
ip_[host] = {}
|
|
for vlan_ in vlans:
|
|
# TODO(pg710r) We need to handle the case when incoming ipv4
|
|
# list is empty
|
|
if len(vlan_.vlan.ipv4) is not 0:
|
|
name = self._get_network_name_from_vlan_name(
|
|
vlan_.vlan.name)
|
|
ipv4 = vlan_.vlan.ipv4[0].ip
|
|
LOG.debug("vlan:{},name:{},ip:{},vlan_name:{}".format(
|
|
vlan_.vlan.vlan_id, name, ipv4, vlan_.vlan.name))
|
|
# TODD(pg710r) This code needs to extended to support ipv4
|
|
# and ipv6
|
|
# ip_[host][name] = {'ipv4': ipv4}
|
|
ip_[host][name] = ipv4
|
|
|
|
return ip_
|
|
|
|
def _get_network_name_from_vlan_name(self, vlan_name):
|
|
""" network names are ksn, oam, oob, overlay, storage, pxe
|
|
|
|
The following mapping rules apply:
|
|
vlan_name contains "ksn" the network name is "calico"
|
|
vlan_name contains "storage" the network name is "storage"
|
|
vlan_name contains "server" the network name is "oam"
|
|
vlan_name contains "ovs" the network name is "overlay"
|
|
vlan_name contains "ILO" the network name is "oob"
|
|
"""
|
|
network_names = {
|
|
'ksn': 'calico',
|
|
'storage': 'storage',
|
|
'server': 'oam',
|
|
'ovs': 'overlay',
|
|
'ILO': 'oob',
|
|
'pxe': 'pxe'
|
|
}
|
|
|
|
for name in network_names:
|
|
# Make a pattern that would ignore case.
|
|
# if name is 'ksn' pattern name is '(?i)(ksn)'
|
|
name_pattern = "(?i)({})".format(name)
|
|
if re.search(name_pattern, vlan_name):
|
|
return network_names[name]
|
|
# Return empty string is vlan_name is not matched with network_names
|
|
return ""
|
|
|
|
def get_dns_servers(self, region):
|
|
try:
|
|
zone = self.region_zone_map[region]['zone']
|
|
zone_id = self._get_zone_id_by_name(zone)
|
|
zone_api = formation_client.ZonesApi(self.formation_api_client)
|
|
zone_ = zone_api.zones_zone_id_get(zone_id)
|
|
except formation_client.rest.ApiException as e:
|
|
raise ApiClientError(e.msg)
|
|
|
|
if not zone_.ipv4_dns:
|
|
LOG.warn("No dns server")
|
|
return []
|
|
|
|
dns_list = []
|
|
for dns in zone_.ipv4_dns:
|
|
dns_list.append(dns.ip)
|
|
|
|
return dns_list
|
|
|
|
def get_ntp_servers(self, region):
|
|
return []
|
|
|
|
def get_ldap_information(self, region):
|
|
return {}
|
|
|
|
def get_location_information(self, region):
|
|
""" get location information for a zone and return """
|
|
site = self.region_zone_map[region]['site']
|
|
site_id = self._get_site_id_by_name(site)
|
|
site_api = formation_client.SitesApi(self.formation_api_client)
|
|
site_info = site_api.sites_site_id_get(site_id)
|
|
|
|
try:
|
|
return {
|
|
# 'corridor': site_info.corridor,
|
|
'name': site_info.city,
|
|
'state': site_info.state,
|
|
'country': site_info.country,
|
|
'physical_location_id': site_info.clli,
|
|
}
|
|
except AttributeError as e:
|
|
raise MissingAttributeError('Missing {} information in {}'.format(
|
|
e, site_info.city))
|
|
|
|
def get_domain_name(self, region):
|
|
try:
|
|
zone = self.region_zone_map[region]['zone']
|
|
zone_id = self._get_zone_id_by_name(zone)
|
|
zone_api = formation_client.ZonesApi(self.formation_api_client)
|
|
zone_ = zone_api.zones_zone_id_get(zone_id)
|
|
except formation_client.rest.ApiException as e:
|
|
raise ApiClientError(e.msg)
|
|
|
|
if not zone_.dns:
|
|
LOG.warn('Got None while running get domain name')
|
|
return None
|
|
|
|
return zone_.dns
|