Base Code for Tugboat Plugin and Addition of config files, templates

This commit is contained in:
Purnendu Ghosh 2018-11-29 02:20:31 +05:30
parent acd81d2b3f
commit 4a8e2720e1
44 changed files with 7917 additions and 78 deletions

3
doc/requirements.txt Normal file
View File

@ -0,0 +1,3 @@
# Documentation
sphinx>=1.6.2
sphinx_rtd_theme==0.2.4

129
doc/source/conf.py Normal file
View File

@ -0,0 +1,129 @@
# -*- coding: utf-8 -*-
#
# shipyard documentation build configuration file, created by
# sphinx-quickstart on Sat Sep 16 03:40:50 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('../../'))
import sphinx_rtd_theme
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.todo',
'sphinx.ext.viewcode',
]
# Add any paths that contain templates here, relative to this directory.
# templates_path = []
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'tugboat'
copyright = u'2018 AT&T Intellectual Property.'
author = u'Tugboat Authors'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u'0.1.0'
# The full version, including alpha/beta/rc tags.
release = u'0.1.0'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = "sphinx_rtd_theme"
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'ucpintdoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}

View File

@ -0,0 +1,132 @@
..
Copyright 2018 AT&T Intellectual Property.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
===============
Getting Started
===============
What is Spyglass?
----------------
Spyglass is a data extraction tool which can interface with
different input data sources to generate site manifest YAML files.
The data sources will provide all the configuration data needed
for a site deployment. These site manifest YAML files generated
by spyglass will be saved in a Git repository, from where Pegleg
can access and aggregate them. This aggregated file can then be
fed to Shipyard for site deployment / updates.
Architecture
------------
::
+-----------+ +-------------+
| | | +-------+ |
| | +------>| |Generic| |
+-----------+ | | | |Object | |
|Tugboat(Xl)| I | | | +-------+ |
|Plugin | N | | | | |
+-----------+ T | | | | |
| E | | | +------+ |
+------------+ R | | | |Parser| +------> Intermediary YAML
|Remote Data | F |---+ | +------+ |
|SourcePlugin| A | | | |
+------------+ C | | |(Intermediary YAML)
| E | | | |
| | | | |
| H | | v |
| A | | +---------+|(templates) +------------+
| N | | |Site |+<--------------|Repository |
| D | | |Processor||-------------->|Adapter |
| L | | +---------+|(Generated +------------+
| E | | ^ | Site Manifests)
| R | | +---|-----+|
| | | | J2 ||
| | | |Templates||
| | | +---------+|
+-----------+ +-------------+
--
Basic Usage
-----------
Before using Spyglass you must:
1. Clone the Spyglass repository:
.. code-block:: console
git clone https://github.com/att-comdev/tugboat/tree/spyglass
2. Install the required packages in spyglass:
.. code-block:: console
pip3 install -r tugboat/requirements.txt
CLI Options
-----------
Usage: spyglass [OPTIONS]
Options:
-s, --site TEXT Specify the site for which manifests to be
generated
-t, --type TEXT Specify the plugin type formation or tugboat
-f, --formation_url TEXT Specify the formation url
-u, --formation_user TEXT Specify the formation user id
-p, --formation_password TEXT Specify the formation user password
-i, --intermediary PATH Intermediary file path generate manifests,
use -m also with this option
-d, --additional_config PATH Site specific configuraton details
-g, --generate_intermediary Dump intermediary file from passed excel and
excel spec
-idir, --intermediary_dir PATH The path where intermediary file needs to be
generated
-e, --edit_intermediary / -nedit, --no_edit_intermediary
Flag to let user edit intermediary
-m, --generate_manifests Generate manifests from the generated
intermediary file
-mdir, --manifest_dir PATH The path where manifest files needs to be
generated
-x, --excel PATH Path to engineering excel file, to be passed
with generate_intermediary
-e, --excel_spec PATH Path to excel spec, to be passed with
generate_intermediary
-l, --loglevel INTEGER Loglevel NOTSET:0 ,DEBUG:10, INFO:20,
WARNING:30, ERROR:40, CRITICAL:50 [default:
20]
--help Show this message and exit.
1. Running Spyglass with Remote Data Source Plugin
spyglass -mg --type formation -f <URL> -u <user_id> -p <password> -d <site_config> -s <sitetype> --template_dir=<j2 template dir>
2. Running Spyglass with Excel Plugin
spyglass -mg --type tugboat -x <Excel File> -e <Excel Spec> -d <Site Config> -s <Region> --template_dir=<j2 template dir>
for example:
spyglass -mg -t tugboat -x SiteDesignSpec_v0.1.xlsx -e excel_spec_upstream.yaml -d site_config.yaml -s airship-seaworthy --template_dir=<j2 template dir>
Where sample 'excel_spec_upstream.yaml', 'SiteDesignSpec_v0.1.xlsx'
'site_config.yaml' and J2 templates can be found under 'spyglass/examples'
folder

34
doc/source/index.rst Normal file
View File

@ -0,0 +1,34 @@
..
Copyright 2018 AT&T Intellectual Property.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
=====================
Spyglass Documentation
=====================
Overview
--------
Spyglass is a data extraction tool which can interface with
different input data sources to generate site manifest YAML files.
The data sources will provide all the configuration data needed
for a site deployment. These site manifest YAML files generated
by spyglass will be saved in a Git repository, from where Pegleg
can access and aggregate them. This aggregated file can then be
fed to Shipyard for site deployment / updates.
.. toctree::
:maxdepth: 2
getting_started

0
spyglass/__init__.py Normal file
View File

View File

View File

@ -0,0 +1,38 @@
###########################
# Global Rules #
###########################
#Rule1: ip_alloc_offset
# Specifies the number of ip addresses to offset from
# the start of subnet allocation pool while allocating it to host.
# -for vlan it is set to 12 as default.
# -for oob it is 10
# -for all gateway ip addresss it is set to 1.
# -for ingress vip it is 1
# -for static end (non pxe) it is -1( means one but last ip of the pool)
# -for dhcp end (pxe only) it is -2( 3rd from the last ip of the pool)
#Rule2: host_profile_interfaces.
# Specifies the network interfaces type and
# and their names for a particular hw profile
#Rule3: hardware_profile
# This specifies the profile details bases on sitetype.
# It specifies the profile name and host type for compute,
# controller along with hw type
---
rule_ip_alloc_offset:
name: ip_alloc_offset
ip_alloc_offset:
default: 12
oob: 10
gateway: 1
ingress_vip: 1
static_ip_end: -2
dhcp_ip_end: -2
rule_hardware_profile:
name: hardware_profile
hardware_profile:
foundry:
profile_name:
compute: dp-r720
ctrl: cp-r720
hw_type: dell_r720
...

View File

@ -277,7 +277,6 @@ class BaseDataSourcePlugin(object):
"""
LOG.info("Extract baremetal information from plugin")
baremetal = {}
is_genesis = False
hosts = self.get_hosts(self.region)
# For each host list fill host profile and network IPs
@ -301,30 +300,19 @@ class BaseDataSourcePlugin(object):
# Fill network IP for this host
temp_host['ip'] = {}
temp_host['ip']['oob'] = temp_host_ips[host_name].get('oob', "")
temp_host['ip']['oob'] = temp_host_ips[host_name].get(
'oob', "#CHANGE_ME")
temp_host['ip']['calico'] = temp_host_ips[host_name].get(
'calico', "")
temp_host['ip']['oam'] = temp_host_ips[host_name].get('oam', "")
'calico', "#CHANGE_ME")
temp_host['ip']['oam'] = temp_host_ips[host_name].get(
'oam', "#CHANGE_ME")
temp_host['ip']['storage'] = temp_host_ips[host_name].get(
'storage', "")
'storage', "#CHANGE_ME")
temp_host['ip']['overlay'] = temp_host_ips[host_name].get(
'overlay', "")
'overlay', "#CHANGE_ME")
temp_host['ip']['pxe'] = temp_host_ips[host_name].get(
'pxe', "#CHANGE_ME")
# Filling rack_type( compute/controller/genesis)
# "cp" host profile is controller
# "ns" host profile is compute
if (temp_host['host_profile'] == 'cp'):
# The controller node is designates as genesis"
if is_genesis is False:
is_genesis = True
temp_host['type'] = 'genesis'
else:
temp_host['type'] = 'controller'
else:
temp_host['type'] = 'compute'
baremetal[rack_name][host_name] = temp_host
LOG.debug("Baremetal information:\n{}".format(
pprint.pformat(baremetal)))
@ -412,8 +400,9 @@ class BaseDataSourcePlugin(object):
for net in networks:
tmp_net = {}
if net['name'] in networks_to_scan:
tmp_net['subnet'] = net['subnet']
tmp_net['vlan'] = net['vlan']
tmp_net['subnet'] = net.get('subnet', '#CHANGE_ME')
if ((net['name'] != 'ingress') and (net['name'] != 'oob')):
tmp_net['vlan'] = net.get('vlan', '#CHANGE_ME')
network_data['vlan_network_data'][net['name']] = tmp_net

View File

@ -433,8 +433,8 @@ class FormationPlugin(BaseDataSourcePlugin):
name_pattern = "(?i)({})".format(name)
if re.search(name_pattern, vlan_name):
return network_names[name]
return ("")
# Return empty string is vlan_name is not matched with network_names
return ""
def get_dns_servers(self, region):
try:

View File

@ -0,0 +1,35 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class BaseError(Exception):
pass
class NotEnoughIp(BaseError):
def __init__(self, cidr, total_nodes):
self.cidr = cidr
self.total_nodes = total_nodes
def display_error(self):
print('{} can not handle {} nodes'.format(self.cidr, self.total_nodes))
class NoSpecMatched(BaseError):
def __init__(self, excel_specs):
self.specs = excel_specs
def display_error(self):
print('No spec matched. Following are the available specs:\n'.format(
self.specs))

View File

@ -0,0 +1,350 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import itertools
import logging
import pprint
import re
from spyglass.data_extractor.base import BaseDataSourcePlugin
from spyglass.data_extractor.plugins.tugboat.excel_parser import ExcelParser
LOG = logging.getLogger(__name__)
class TugboatPlugin(BaseDataSourcePlugin):
def __init__(self, region):
LOG.info("Tugboat Initializing")
self.source_type = 'excel'
self.source_name = 'tugboat'
# Configuration parameters
self.excel_path = None
self.excel_spec = None
# Site related data
self.region = region
# Raw data from excel
self.parsed_xl_data = None
LOG.info("Initiated data extractor plugin:{}".format(self.source_name))
def set_config_opts(self, conf):
"""
Placeholder to set confgiuration options
specific to each plugin.
:param dict conf: Configuration options as dict
Example: conf = { 'excel_spec': 'spec1.yaml',
'excel_path': 'excel.xls' }
Each plugin will have their own config opts.
"""
self.excel_path = conf['excel_path']
self.excel_spec = conf['excel_spec']
# Extract raw data from excel sheets
self._get_excel_obj()
self._extract_raw_data_from_excel()
return
def get_plugin_conf(self, kwargs):
""" Validates the plugin param from CLI and return if correct
Ideally the CLICK module shall report an error if excel file
and excel specs are not specified. The below code has been
written as an additional safeguard.
"""
try:
assert (len(
kwargs['excel'])), "Engineering Spec file not specified"
excel_file_info = kwargs['excel']
assert (kwargs['excel_spec']
) is not None, "Excel Spec file not specified"
excel_spec_info = kwargs['excel_spec']
except AssertionError as e:
LOG.error("{}:Spyglass exited!".format(e))
exit()
plugin_conf = {
'excel_path': excel_file_info,
'excel_spec': excel_spec_info
}
return plugin_conf
def get_hosts(self, region, rack=None):
"""Return list of hosts in the region
:param string region: Region name
:param string rack: Rack name
:returns: list of hosts information
:rtype: list of dict
Example: [
{
'name': 'host01',
'type': 'controller',
'host_profile': 'hp_01'
},
{
'name': 'host02',
'type': 'compute',
'host_profile': 'hp_02'}
]
"""
LOG.info("Get Host Information")
ipmi_data = self.parsed_xl_data['ipmi_data'][0]
rackwise_hosts = self._get_rackwise_hosts()
host_list = []
for rack in rackwise_hosts.keys():
for host in rackwise_hosts[rack]:
host_list.append({
'rack_name':
rack,
'name':
host,
'host_profile':
ipmi_data[host]['host_profile']
})
return host_list
def get_networks(self, region):
""" Extracts vlan network info from raw network data from excel"""
vlan_list = []
# Network data extracted from xl is formatted to have a predictable
# data type. For e.g VlAN 45 extracted from xl is formatted as 45
vlan_pattern = r'\d+'
private_net = self.parsed_xl_data['network_data']['private']
public_net = self.parsed_xl_data['network_data']['public']
# Extract network information from private and public network data
for net_type, net_val in itertools.chain(private_net.items(),
public_net.items()):
tmp_vlan = {}
# Ingress is special network that has no vlan, only a subnet string
# So treatment for ingress is different
if net_type is not 'ingress':
# standardize the network name as net_type may ne different.
# For e.g insteas of pxe it may be PXE or instead of calico
# it may be ksn. Valid network names are pxe, calico, oob, oam,
# overlay, storage, ingress
tmp_vlan['name'] = self._get_network_name_from_vlan_name(
net_type)
# extract vlan tag. It was extracted from xl file as 'VlAN 45'
# The code below extracts the numeric data fron net_val['vlan']
if net_val.get('vlan', "") is not "":
value = re.findall(vlan_pattern, net_val['vlan'])
tmp_vlan['vlan'] = value[0]
else:
tmp_vlan['vlan'] = "#CHANGE_ME"
tmp_vlan['subnet'] = net_val.get('subnet', "#CHANGE_ME")
tmp_vlan['gateway'] = net_val.get('gateway', "#CHANGE_ME")
else:
tmp_vlan['name'] = 'ingress'
tmp_vlan['subnet'] = net_val
vlan_list.append(tmp_vlan)
LOG.debug("vlan list extracted from tugboat:\n{}".format(
pprint.pformat(vlan_list)))
return vlan_list
def get_ips(self, region, host=None):
"""Return list of IPs on the host
:param string region: Region name
:param string host: Host name
:returns: Dict of IPs per network on the host
:rtype: dict
Example: {'oob': {'ipv4': '192.168.1.10'},
'pxe': {'ipv4': '192.168.2.10'}}
The network name from get_networks is expected to be the keys of this
dict. In case some networks are missed, they are expected to be either
DHCP or internally generated n the next steps by the design rules.
"""
ip_ = {}
ipmi_data = self.parsed_xl_data['ipmi_data'][0]
ip_[host] = {
'oob': ipmi_data[host].get('ipmi_address', '#CHANGE_ME'),
'oam': ipmi_data[host].get('oam', '#CHANGE_ME'),
'calico': ipmi_data[host].get('calico', '#CHANGE_ME'),
'overlay': ipmi_data[host].get('overlay', '#CHANGE_ME'),
'pxe': ipmi_data[host].get('pxe', '#CHANGE_ME'),
'storage': ipmi_data[host].get('storage', '#CHANGE_ME')
}
return ip_
def get_ldap_information(self, region):
""" Extract ldap information from excel"""
ldap_raw_data = self.parsed_xl_data['site_info']['ldap']
ldap_info = {}
# raw url is 'url: ldap://example.com' so we are converting to
# 'ldap://example.com'
url = ldap_raw_data.get('url', '#CHANGE_ME')
try:
ldap_info['url'] = url.split(' ')[1]
ldap_info['domain'] = url.split('.')[1]
except IndexError as e:
LOG.error("url.split:{}".format(e))
ldap_info['common_name'] = ldap_raw_data.get('common_name',
'#CHANGE_ME')
ldap_info['subdomain'] = ldap_raw_data.get('subdomain', '#CHANGE_ME')
return ldap_info
def get_ntp_servers(self, region):
""" Returns a comma separated list of ntp ip addresses"""
ntp_server_list = self._get_formatted_server_list(
self.parsed_xl_data['site_info']['ntp'])
return ntp_server_list
def get_dns_servers(self, region):
""" Returns a comma separated list of dns ip addresses"""
dns_server_list = self._get_formatted_server_list(
self.parsed_xl_data['site_info']['dns'])
return dns_server_list
def get_domain_name(self, region):
""" Returns domain name extracted from excel file"""
return self.parsed_xl_data['site_info']['domain']
def get_location_information(self, region):
"""
Prepare location data from information extracted
by ExcelParser(i.e raw data)
"""
location_data = self.parsed_xl_data['site_info']['location']
corridor_pattern = r'\d+'
corridor_number = re.findall(corridor_pattern,
location_data['corridor'])[0]
name = location_data.get('name', '#CHANGE_ME')
state = location_data.get('state', '#CHANGE_ME')
country = location_data.get('country', '#CHANGE_ME')
physical_location_id = location_data.get('physical_location', '')
return {
'name': name,
'physical_location_id': physical_location_id,
'state': state,
'country': country,
'corridor': 'c{}'.format(corridor_number),
}
def get_racks(self, region):
# This function is not required since the excel plugin
# already provide rack information.
pass
def _get_excel_obj(self):
""" Creation of an ExcelParser object to store site information.
The information is obtained based on a excel spec yaml file.
This spec contains row, column and sheet information of
the excel file from where site specific data can be extracted.
"""
self.excel_obj = ExcelParser(self.excel_path, self.excel_spec)
def _extract_raw_data_from_excel(self):
""" Extracts raw information from excel file based on excel spec"""
self.parsed_xl_data = self.excel_obj.get_data()
def _get_network_name_from_vlan_name(self, vlan_name):
""" network names are ksn, oam, oob, overlay, storage, pxe
This is a utility function to determine the vlan acceptable
vlan from the name extracted from excel file
The following mapping rules apply:
vlan_name contains "ksn or calico" the network name is "calico"
vlan_name contains "storage" the network name is "storage"
vlan_name contains "server" the network name is "oam"
vlan_name contains "ovs" the network name is "overlay"
vlan_name contains "oob" the network name is "oob"
vlan_name contains "pxe" the network name is "pxe"
"""
network_names = [
'ksn|calico', 'storage', 'oam|server', 'ovs|overlay', 'oob', 'pxe'
]
for name in network_names:
# Make a pattern that would ignore case.
# if name is 'ksn' pattern name is '(?i)(ksn)'
name_pattern = "(?i)({})".format(name)
if re.search(name_pattern, vlan_name):
if name is 'ksn|calico':
return 'calico'
if name is 'storage':
return 'storage'
if name is 'oam|server':
return 'oam'
if name is 'ovs|overlay':
return 'overlay'
if name is 'oob':
return 'oob'
if name is 'pxe':
return 'pxe'
# if nothing matches
LOG.error(
"Unable to recognize VLAN name extracted from Plugin data source")
return ("")
def _get_formatted_server_list(self, server_list):
""" Format dns and ntp server list as comma separated string """
# dns/ntp server info from excel is of the format
# 'xxx.xxx.xxx.xxx, (aaa.bbb.ccc.com)'
# The function returns a list of comma separated dns ip addresses
servers = []
for data in server_list:
if '(' not in data:
servers.append(data)
formatted_server_list = ','.join(servers)
return formatted_server_list
def _get_rack(self, host):
"""
Get rack id from the rack string extracted
from xl
"""
rack_pattern = r'\w.*(r\d+)\w.*'
rack = re.findall(rack_pattern, host)[0]
if not self.region:
self.region = host.split(rack)[0]
return rack
def _get_rackwise_hosts(self):
""" Mapping hosts with rack ids """
rackwise_hosts = {}
hostnames = self.parsed_xl_data['ipmi_data'][1]
racks = self._get_rack_data()
for rack in racks:
if rack not in rackwise_hosts:
rackwise_hosts[racks[rack]] = []
for host in hostnames:
if rack in host:
rackwise_hosts[racks[rack]].append(host)
LOG.debug("rackwise hosts:\n%s", pprint.pformat(rackwise_hosts))
return rackwise_hosts
def _get_rack_data(self):
""" Format rack name """
LOG.info("Getting rack data")
racks = {}
hostnames = self.parsed_xl_data['ipmi_data'][1]
for host in hostnames:
rack = self._get_rack(host)
racks[rack] = rack.replace('r', 'rack')
return racks

Binary file not shown.

View File

@ -0,0 +1,63 @@
# Copyright 2018 The Openstack-Helm Authors.
# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Important: Please modify the dictionary with appropriate
# design spec file.
---
specs:
# Design Spec file name: SiteDesignSpec_v0.1.xlsx
xl_spec:
ipmi_sheet_name: 'Site-Information'
start_row: 4
end_row: 15
hostname_col: 2
ipmi_address_col: 3
host_profile_col: 5
ipmi_gateway_col: 4
private_ip_sheet: 'Site-Information'
net_type_col: 1
vlan_col: 2
vlan_start_row: 19
vlan_end_row: 30
net_start_row: 33
net_end_row: 40
net_col: 2
net_vlan_col: 1
public_ip_sheet: 'Site-Information'
oam_vlan_col: 1
oam_ip_row: 43
oam_ip_col: 2
oob_net_row: 48
oob_net_start_col: 2
oob_net_end_col: 5
ingress_ip_row: 45
dns_ntp_ldap_sheet: 'Site-Information'
login_domain_row: 52
ldap_col: 2
global_group: 53
ldap_search_url_row: 54
ntp_row: 55
ntp_col: 2
dns_row: 56
dns_col: 2
domain_row: 51
domain_col: 2
location_sheet: 'Site-Information'
column: 2
corridor_row: 59
site_name_row: 58
state_name_row: 60
country_name_row: 61
clli_name_row: 62

View File

@ -0,0 +1,33 @@
##################################
# Site Specific Tugboat Settings #
##################################
---
site_info:
ldap:
common_name: test
url: ldap://ldap.example.com
subdomain: test
ntp:
servers: 10.10.10.10,20.20.20.20,30.30.30.30
sitetype: foundry
domain: atlantafoundry.com
dns:
servers: 8.8.8.8,8.8.4.4,208.67.222.222
network:
vlan_network_data:
ingress:
subnet:
- 132.68.226.72/29
bgp :
peers:
- '172.29.0.2'
- '172.29.0.3'
asnumber: 64671
peer_asnumber: 64688
storage:
ceph:
controller:
osd_count: 6
...

View File

@ -0,0 +1,26 @@
---
schema: 'drydock/BootAction/v1'
metadata:
schema: 'metadata/Document/v1'
name: promjoin
storagePolicy: 'cleartext'
layeringDefinition:
abstract: false
layer: site
labels:
application: 'drydock'
data:
signaling: false
assets:
- path: /opt/promjoin.sh
type: file
permissions: '555'
{% raw %}
location: promenade+http://promenade-api.ucp.svc.cluster.local/api/v1.0/join-scripts?design_ref={{ action.design_ref | urlencode }}&hostname={{ node.hostname }}&ip={{ node.network.calico.ip }}{% endif %}{% for k, v in node.labels.items() %}&labels.dynamic={{ k }}={{ v }}{% endfor %}
{% endraw %}
location_pipeline:
- template
data_pipeline:
- utf8_decode
...

View File

@ -0,0 +1,51 @@
{% set control_count = [1] %}
{% for rack in data['baremetal'].keys() %}
{% for host in data['baremetal'][rack].keys()%}
{% if data['baremetal'][rack][host]['type'] != 'genesis' %}
---
schema: 'drydock/BaremetalNode/v1'
metadata:
schema: 'metadata/Document/v1'
name: {{ host }}
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
oob:
account: 'root'
{% if data['baremetal'][rack][host]['host_profile'] == 'cp' %}
{% if control_count.append(control_count.pop()+1) %} {% endif %}
{% if control_count[0] < 4 %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-primary
{% else %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-secondary
{% endif %}
{% else %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}
{% endif %}
addressing:
- network: oob
address: {{ data['baremetal'][rack][host]['ip']['oob'] }}
- network: oam
address: {{ data['baremetal'][rack][host]['ip']['oam'] }}
- network: pxe
address: {{ data['baremetal'][rack][host]['ip']['pxe'] }}
- network: storage
address: {{ data['baremetal'][rack][host]['ip']['storage'] }}
- network: calico
address: {{ data['baremetal'][rack][host]['ip']['calico'] }}
- network: overlay
address: {{ data['baremetal'][rack][host]['ip']['overlay'] }}
metadata:
rack: RACK{{rack[-2:] }}
tags:
{% if data['baremetal'][rack][host]['type'] == 'compute' %}
- 'workers'
{% else %}
- 'masters'
{% endif %}
...
{% endif %}
{%endfor%}
{%endfor%}

View File

@ -0,0 +1,90 @@
---
# The purpose of this file is to provide Shipyard a strategy to aid in the site's
# deployment. This WILL require modification for each particular site. A successful
# strategy for large labs that has been used in the past has been to split the Control
# Plane hosts up from the computes, as well as the computes by rack. The below strategy
# differs slightly, as the size of the lab is smaller. As such, the Control Plane hosts
# deploy first, followed by half of the computes, followed by the second half of the
# computes. Shipyard deployment strategies can be very useful in getting around certain
# failures, like misbehaving nodes that may hold up the deployment. See more at:
# https://github.com/openstack/airship-shipyard/blob/master/doc/source/site-definition-documents.rst#deployment-strategy
schema: shipyard/DeploymentStrategy/v1
metadata:
schema: metadata/Document/v1
replacement: true
name: deployment-strategy
layeringDefinition:
abstract: false
layer: site
parentSelector:
name: deployment-strategy-global
actions:
- method: replace
path: .
storagePolicy: cleartext
replacement: true
data:
groups:
- name: masters
critical: true
depends_on: []
selectors:
- node_names: []
node_labels: []
node_tags:
- masters
rack_names: []
success_criteria:
percent_successful_nodes: 100
# NEWSITE-CHANGEME: The number of "worker groups" should equal the number of site racks
- name: worker_group_0
critical: false
depends_on:
- masters
selectors:
# NEWSITE-CHANGEME: The following should be a list of the computes in the site's first rack
- node_names:
{% for rack in data['baremetal'].keys() %}
{% for host in data['baremetal'][rack].keys()%}
{% if rack == 'rack03' or rack == 'rack04' %}
- {{ host }}
{% endif %}
{% endfor %}
{% endfor %}
node_labels: []
node_tags: []
rack_names: []
- name: worker_group_1
critical: false
depends_on:
- masters
selectors:
# NEWSITE-CHANGEME: The following should be a list of the computes in the site's second rack
- node_names:
{% for rack in data['baremetal'].keys() %}
{% for host in data['baremetal'][rack].keys()%}
{% if rack == 'rack05' or rack == 'rack06' %}
- {{ host }}
{% endif %}
{% endfor %}
{% endfor %}
node_labels: []
node_tags: []
rack_names: []
- name: workers
critical: true
# NEWSITE-CHANGEME: Populate with each worker group (should equal the number of site racks).
# This group ensures a percent of success is achieved with the compute deployments.
depends_on:
- worker_group_0
- worker_group_1
selectors:
- node_names: []
node_labels: []
node_tags:
- workers
rack_names: []
success_criteria:
percent_successful_nodes: 60
...

View File

@ -0,0 +1,107 @@
---
schema: pegleg/CommonAddresses/v1
metadata:
schema: metadata/Document/v1
name: common-addresses
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
calico:
ip_autodetection_method: interface=bond1.{{ data['network']['vlan_network_data']['calico']['vlan']}}
etcd:
service_ip: 10.96.232.136
ip_rule:
gateway: {{ data['network']['vlan_network_data']['calico']['gateway']}}
overlap_cidr: 10.96.0.0/15
bgp:
ipv4:
public_service_cidr: {{ data['network']['vlan_network_data']['ingress']['subnet'][0] }}
ingress_vip: {{ data['network']['bgp']['ingress_vip'] }}
peers:
{% for peer in data['network']['bgp']['peers'] %}
- {{ peer }}
{% endfor %}
dns:
cluster_domain: cluster.local
service_ip: 10.96.0.10
upstream_servers:
{% for server in (data['site_info']['dns']['servers']).split(',') %}
- {{ server }}
{% endfor %}
upstream_servers_joined: {{ data['site_info']['dns']['servers']}}
ingress_domain: {{ data['site_info']['domain']|lower }}
genesis:
hostname: {{ (data|get_role_wise_nodes)['genesis']['name'] }}
{% for rack in data['baremetal'] %}
{% for host in data['baremetal'][rack] %}
{% if data['baremetal'][rack][host]['type'] == 'genesis' %}
ip: {{ data['baremetal'][rack][host]['ip']['calico'] }}
{% endif %}
{% endfor %}
{% endfor %}
bootstrap:
ip: {{ (data|get_role_wise_nodes)['genesis']['pxe'] }}
kubernetes:
api_service_ip: 10.96.0.1
etcd_service_ip: 10.96.0.2
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
# misc k8s port settings
apiserver_port: 6443
haproxy_port: 6553
service_node_port_range: 30000-32767
# etcd port settings
etcd:
container_port: 2379
haproxy_port: 2378
masters:
{% for host in (data|get_role_wise_nodes)['masters'] %}
- hostname: {{ host }}
{% endfor %}
# NEWSITE-CHANGEME: Environment proxy information.
# NOTE: Reference Airship sites do not deploy behind a proxy, so this proxy section
# should be commented out.
# However if you are in a lab that requires proxy, ensure that these proxy
# settings are correct and reachable in your environment; otherwise update
# them with the correct values for your environment.
proxy:
http: ""
https: ""
no_proxy: []
node_ports:
drydock_api: 30000
maas_api: 30001
maas_proxy: 31800 # hardcoded in MAAS
shipyard_api: 30003
airflow_web: 30004
ntp:
servers_joined: {{ data['site_info']['ntp']['servers'] }}
ldap:
base_url: {{ (data['site_info']['ldap']['url']|string).split('//')[1] }}
url: {{ data['site_info']['ldap']['url'] }}
auth_path: DC=test,DC=test,DC=com?sAMAccountName?sub?memberof=CN={{ data['site_info']['ldap']['common_name'] }},OU=Application,OU=Groups,DC=test,DC=test,DC=com
common_name: {{ data['site_info']['ldap']['common_name'] }}
subdomain: {{ data['site_info']['ldap']['subdomain'] }}
domain: {{ (data['site_info']['ldap']['url']|string).split('.')[1] }}
storage:
ceph:
public_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
cluster_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
neutron:
tunnel_device: 'bond1.{{ data['network']['vlan_network_data']['overlay']['vlan'] }}'
external_iface: 'bond1'
openvswitch:
external_iface: 'bond1'
...

View File

@ -0,0 +1,251 @@
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: oob
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
# MaaS doesnt own this network like it does the others, so the noconfig label
# is specified.
labels:
noconfig: enabled
bonding:
mode: disabled
mtu: 1500
linkspeed: auto
trunking:
mode: disabled
default_network: oob
allowed_networks:
- oob
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: oob
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['oob']['subnet'] }}
routes:
- subnet: '0.0.0.0/0'
gateway: {{ data['network']['vlan_network_data']['oob']['gateway'] }}
metric: 100
ranges:
- type: static
start: {{ data['network']['vlan_network_data']['oob']['static_start'] }}
end: {{ data['network']['vlan_network_data']['oob']['static_end'] }}
...
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: pxe
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
bonding:
mode: disabled
mtu: 1500
linkspeed: auto
trunking:
mode: disabled
default_network: pxe
allowed_networks:
- pxe
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: pxe
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: pxe
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['pxe']['subnet'] }}
routes:
{% for other_subnet in data['network']['vlan_network_data']['pxe']['routes'] %}
- subnet: {{ other_subnet }}
gateway: {{ data['network']['vlan_network_data']['pxe']['gateway'] }}
metric: 100
{% endfor %}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['pxe']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['pxe']['reserved_end'] }}
- type: static
start: {{ data['network']['vlan_network_data']['pxe']['static_start'] }}
end: {{ data['network']['vlan_network_data']['pxe']['static_end'] }}
- type: dhcp
start: {{ data['network']['vlan_network_data']['pxe']['dhcp_start'] }}
end: {{ data['network']['vlan_network_data']['pxe']['dhcp_end'] }}
...
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: data
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
bonding:
mode: 802.3ad
hash: layer3+4
peer_rate: fast
mon_rate: 100
up_delay: 1000
down_delay: 3000
# NEWSITE-CHANGEME: Ensure the network switches in the environment are
# configured for this MTU or greater. Even if switches are configured for or
# can support a slightly higher MTU, there is no need (and negliable benefit)
# to squeeze every last byte into the MTU (e.g., 9216 vs 9100). Leave MTU at
# 9100 for maximum compatibility.
mtu: 9100
linkspeed: auto
trunking:
mode: 802.1q
allowed_networks:
- oam
- storage
- overlay
- calico
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: oam
layeringDefinition:
abstract: false
layer: 'site'
parentSelector:
network_role: oam
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['oam']['subnet'] }}
{% set flag = [0] %}
{% for route in data['network']['vlan_network_data']['oam']['routes'] %}
{% if flag[0] == 0 %}
routes:
{% endif %}
{% if flag.append(flag.pop() + 1) %} {% endif %}
- subnet: {{ route }}
gateway: {{ data['network']['vlan_network_data']['oam']['gateway'] }}
metric: 100
{% endfor %}
{% if flag[0] == 0 %}
routes:[]
{% endif %}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['oam']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['oam']['reserved_end'] }}
- type: static
start: {{ data['network']['vlan_network_data']['oam']['static_start'] }}
end: {{ data['network']['vlan_network_data']['oam']['static_end'] }}
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: storage
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: storage
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['storage']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['storage']['reserved_end'] }}
- type: static
start: {{ data['network']['vlan_network_data']['storage']['static_start'] }}
end: {{ data['network']['vlan_network_data']['storage']['static_end'] }}
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: overlay
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: os-overlay
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['overlay']['subnet'] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['overlay']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['overlay']['reserved_end'] }}
- type: static
start: {{ data['network']['vlan_network_data']['overlay']['static_start'] }}
end: {{ data['network']['vlan_network_data']['overlay']['static_end'] }}
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: calico
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: calico
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['calico']['subnet'] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['calico']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['calico']['reserved_end'] }}
- type: static
start: {{ data['network']['vlan_network_data']['calico']['static_start'] }}
end: {{ data['network']['vlan_network_data']['calico']['static_end'] }}
...

View File

@ -0,0 +1,187 @@
---
schema: promenade/PKICatalog/v1
metadata:
schema: metadata/Document/v1
name: cluster-certificates
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
certificate_authorities:
kubernetes:
description: CA for Kubernetes components
certificates:
- document_name: apiserver
description: Service certificate for Kubernetes apiserver
common_name: apiserver
hosts:
- localhost
- 127.0.0.1
- 10.96.0.1
kubernetes_service_names:
- kubernetes.default.svc.cluster.local
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: kubelet-genesis
common_name: system:node:{{ host }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}}
groups:
- system:nodes
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
- document_name: kubelet-{{ host }}
common_name: system:node:{{ host }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}}
groups:
- system:nodes
{%endfor%}
{%endfor%}
- document_name: scheduler
description: Service certificate for Kubernetes scheduler
common_name: system:kube-scheduler
- document_name: controller-manager
description: certificate for controller-manager
common_name: system:kube-controller-manager
- document_name: admin
common_name: admin
groups:
- system:masters
- document_name: armada
common_name: armada
groups:
- system:masters
kubernetes-etcd:
description: Certificates for Kubernetes's etcd servers
certificates:
- document_name: apiserver-etcd
description: etcd client certificate for use by Kubernetes apiserver
common_name: apiserver
# NOTE(mark-burnett): hosts not required for client certificates
- document_name: kubernetes-etcd-anchor
description: anchor
common_name: anchor
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: kubernetes-etcd-genesis
common_name: kubernetes-etcd-genesis
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis'%}
- document_name: kubernetes-etcd-{{ host }}
common_name: kubernetes-etcd-{{ host }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
kubernetes-etcd-peer:
certificates:
- document_name: kubernetes-etcd-genesis-peer
common_name: kubernetes-etcd-genesis-peer
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: kubernetes-etcd-{{ host }}-peer
common_name: kubernetes-etcd-{{ host }}-peer
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}