airship
/
spyglass
Code Issues Proposed changes

Merge "Use data objects for document generation"

This commit is contained in:
Zuul 2019-07-01 13:51:37 +00:00 committed by Gerrit Code Review
parent 746d7ca2ec efe24d8a5f
commit ca9756de09
18 changed files with 711 additions and 558 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spyglass/data_extractor/custom_exceptions.py
View File

@ -59,3 +59,7 @@ class TokenGenerationError(BaseError):
class FormationConnectionError(BaseError):
pass
class InvalidIntermediary(BaseError):
pass

112
spyglass/data_extractor/models.py
View File

@ -12,9 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from copy import deepcopy
import ipaddress
import logging
from spyglass.data_extractor.custom_exceptions import InvalidIntermediary
DATA_DEFAULT = "#CHANGE_ME"
LOG = logging.getLogger(__name__)
@ -253,6 +256,19 @@ class Rack(object):
return host
return None
def get_host_by_type(self, host_type: str):
"""Gets host(s) on rack by role
:param host_type: Role of the host(s) to be retrieved
:return: list of hosts
:rtype: list
"""
matching_hosts = []
for host in self.hosts:
if host.type == host_type:
matching_hosts.append(host)
return matching_hosts
class VLANNetworkData(object):
"""Model for single entry of VLAN Network Data"""
@ -594,3 +610,99 @@ class SiteDocumentData(object):
if rack.name == name:
return rack
return None
def get_baremetal_host_by_type(self, *args):
"""Return baremetal host(s) with matching type
:param args: type(s) of the baremetal host
:return: Host object(s) matching the specified host_type
:rtype: list
"""
host_list = []
for rack in self.baremetal:
rack_hosts = []
for arg in args:
rack_hosts.extend(rack.get_host_by_type(arg))
host_list.extend(rack_hosts)
return host_list
def _validate_key_in_intermediary_dict(key: str, dictionary: dict):
if key not in dictionary:
raise InvalidIntermediary(
'%s is not defined in the given intermediary file.' % key)
def site_document_data_factory(intermediary_dict: dict) -> SiteDocumentData:
"""Uses intermediary file data to create a SiteDocumentData object
:param intermediary_dict: A loaded intermediary file dictionary
:return: all intermediary dictionary data returned as an object
"""
# Validate baremetal in intermediary
_validate_key_in_intermediary_dict('baremetal', intermediary_dict)
# Pull out baremetal data into Rack and Host objects
rack_list = []
for rack, hosts in intermediary_dict['baremetal'].items():
host_list = []
for host_name, host_data in hosts.items():
host_ip_list = IPList(**host_data['ip'])
host_kwargs = {
'rack_name': rack,
'host_profile': host_data['host_profile'],
'type': host_data['type'],
'ip': host_ip_list
}
new_host = Host(host_name, **host_kwargs)
host_list.append(new_host)
new_rack = Rack(rack, host_list)
rack_list.append(new_rack)
# Validate network in intermediary
_validate_key_in_intermediary_dict('network', intermediary_dict)
# Validate vlan_network_data in intermediary
_validate_key_in_intermediary_dict(
'vlan_network_data', intermediary_dict['network'])
# Validate bgp in intermediary
_validate_key_in_intermediary_dict('bgp', intermediary_dict['network'])
# Pull out network data into Network object
vlan_data_list = []
for network_type, network_data in \
intermediary_dict['network']['vlan_network_data'].items():
vlan_data_list.append(VLANNetworkData(network_type, **network_data))
network = Network(vlan_data_list, bgp=intermediary_dict['network']['bgp'])
# Validate site_info in intermediary
_validate_key_in_intermediary_dict('site_info', intermediary_dict)
# Validate dns in intermediary
_validate_key_in_intermediary_dict('dns', intermediary_dict['site_info'])
# Validate ntp in intermediary
_validate_key_in_intermediary_dict('ntp', intermediary_dict['site_info'])
# Validate region_name in intermediary
_validate_key_in_intermediary_dict('region_name', intermediary_dict)
# Pull out site_info into a SiteInfo object
dns_server_list = ServerList(
intermediary_dict['site_info']['dns']['servers'].split(','))
ntp_server_list = ServerList(
intermediary_dict['site_info']['ntp']['servers'].split(','))
site_info_dict = deepcopy(intermediary_dict['site_info'])
site_info_dict.pop('dns')
site_info_dict.pop('ntp')
site_info_dict['dns'] = dns_server_list
site_info_dict['ntp'] = ntp_server_list
site_info_dict['region_name'] = intermediary_dict['region_name']
site_info = SiteInfo(**site_info_dict)
# Validate storage in intermediary
_validate_key_in_intermediary_dict('storage', intermediary_dict)
# Create and return SiteDocumentData object
site_document_data = SiteDocumentData(
site_info=site_info,
network=network,
baremetal=rack_list,
storage=intermediary_dict['storage'])
return site_document_data

26
spyglass/examples/templates/baremetal/bootactions/promjoin.yaml.j2
View File

@ -1,26 +0,0 @@
---
schema: 'drydock/BootAction/v1'
metadata:
schema: 'metadata/Document/v1'
name: promjoin
storagePolicy: 'cleartext'
layeringDefinition:
abstract: false
layer: site
labels:
application: 'drydock'
data:
signaling: false
assets:
- path: /opt/promjoin.sh
type: file
permissions: '555'
{% raw %}
location: promenade+http://promenade-api.ucp.svc.cluster.local/api/v1.0/join-scripts?design_ref={{ action.design_ref | urlencode }}&hostname={{ node.hostname }}&ip={{ node.network.calico.ip }}{% endif %}{% for k, v in node.labels.items() %}&labels.dynamic={{ k }}={{ v }}{% endfor %}
{% endraw %}
location_pipeline:
- template
data_pipeline:
- utf8_decode
...

54
spyglass/examples/templates/baremetal/nodes.yaml.j2
View File

@ -1,50 +1,46 @@
{% set control_count = [1] %}
{% for rack in data['baremetal'].keys() %}
{% for host in data['baremetal'][rack].keys()%}
{% if data['baremetal'][rack][host]['type'] != 'genesis' %}
{% for rack in data.baremetal %}
{% for host in rack.hosts %}
{% if host.type != 'genesis' %}
---
schema: 'drydock/BaremetalNode/v1'
metadata:
schema: 'metadata/Document/v1'
name: {{ host }}
name: {{ host.name }}
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
oob:
account: 'root'
{% if data['baremetal'][rack][host]['host_profile'] == 'cp' %}
{% if control_count.append(control_count.pop()+1) %} {% endif %}
{% if control_count[0] < 4 %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-primary
{% else %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-secondary
{% endif %}
{% else %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}
{% endif %}
{% if host.host_profile == 'cp' %}
{% if loop.index - 1 < 4 %}
host_profile: nc-{{ host.host_profile }}-primary
{% else %}
host_profile: nc-{{ host.host_profile }}-secondary
{% endif %}
{% else %}
host_profile: nc-{{ host.host_profile }}
{% endif %}
addressing:
- network: oob
address: {{ data['baremetal'][rack][host]['ip']['oob'] }}
address: {{ host.ip.oob }}
- network: oam
address: {{ data['baremetal'][rack][host]['ip']['oam'] }}
address: {{ host.ip.oam }}
- network: pxe
address: {{ data['baremetal'][rack][host]['ip']['pxe'] }}
address: {{ host.ip.pxe }}
- network: storage
address: {{ data['baremetal'][rack][host]['ip']['storage'] }}
address: {{ host.ip.storage }}
- network: calico
address: {{ data['baremetal'][rack][host]['ip']['calico'] }}
address: {{ host.ip.calico }}
- network: overlay
address: {{ data['baremetal'][rack][host]['ip']['overlay'] }}
address: {{ host.ip.overlay }}
metadata:
rack: RACK{{rack[-2:] }}
rack: {{ rack.name }}
tags:
{% if data['baremetal'][rack][host]['type'] == 'compute' %}
- 'workers'
{% else %}
- 'masters'
{% endif %}
{% if host.type == 'compute' %}
- 'workers'
{% else %}
- 'masters'
{% endif %}
...
{% endif %}
{%endfor%}

29
spyglass/examples/templates/deployment/deployment-strategy.yaml.j2
View File

@ -11,7 +11,6 @@
schema: shipyard/DeploymentStrategy/v1
metadata:
schema: metadata/Document/v1
replacement: true
name: deployment-strategy
layeringDefinition:
abstract: false
@ -44,13 +43,13 @@ data:
selectors:
# NEWSITE-CHANGEME: The following should be a list of the computes in the site's first rack
- node_names:
{% for rack in data['baremetal'].keys() %}
{% for host in data['baremetal'][rack].keys()%}
{% if rack == 'rack03' or rack == 'rack04' %}
- {{ host }}
{% endif %}
{% endfor %}
{% endfor %}
{% for rack in data.baremetal %}
{% for host in rack.hosts %}
{% if rack.name == 'rack03' or rack.name == 'rack04' %}
- {{ host.name }}
{% endif %}
{% endfor %}
{% endfor %}
node_labels: []
node_tags: []
rack_names: []
@ -61,13 +60,13 @@ data:
selectors:
# NEWSITE-CHANGEME: The following should be a list of the computes in the site's second rack
- node_names:
{% for rack in data['baremetal'].keys() %}
{% for host in data['baremetal'][rack].keys()%}
{% if rack == 'rack05' or rack == 'rack06' %}
- {{ host }}
{% endif %}
{% endfor %}
{% endfor %}
{% for rack in data.baremetal %}
{% for host in rack.hosts %}
{% if rack.name == 'rack05' or rack.name == 'rack06' %}
- {{ host.name }}
{% endif %}
{% endfor %}
{% endfor %}
node_labels: []
node_tags: []
rack_names: []

95
spyglass/examples/templates/networks/common_addresses.yaml.j2
View File

@ -6,102 +6,69 @@ metadata:
layeringDefinition:
abstract: false
layer: site
parentSelector:
name: common-addresses-global
actions:
- method: replace
path: .dns.upstream_servers
- method: merge
path: .
storagePolicy: cleartext
replacement: true
data:
calico:
ip_autodetection_method: interface=bond1.{{ data['network']['vlan_network_data']['calico']['vlan']}}
ip_autodetection_method: interface=bond1.{{ data.network.get_vlan_data_by_name('calico').vlan }}
etcd:
service_ip: 10.96.232.136
ip_rule:
gateway: {{ data['network']['vlan_network_data']['calico']['gateway']}}
gateway: {{ data.network.get_vlan_data_by_name('calico').gateway }}
overlap_cidr: 10.96.0.0/15
bgp:
ipv4:
public_service_cidr: {{ data['network']['vlan_network_data']['ingress']['subnet'][0] }}
ingress_vip: {{ data['network']['bgp']['ingress_vip'] }}
public_service_cidr: {{ data.network.get_vlan_data_by_name('ingress').subnet[0] }}
ingress_vip: {{ data.network.bgp['ingress_vip'] }}
peers:
{% for peer in data['network']['bgp']['peers'] %}
{% for peer in data.network.bgp['peers'] %}
- {{ peer }}
{% endfor %}
{% endfor %}
dns:
cluster_domain: cluster.local
service_ip: 10.96.0.10
upstream_servers:
{% for server in (data['site_info']['dns']['servers']).split(',') %}
{% for server in data.site_info.dns.servers %}
- {{ server }}
{% endfor %}
upstream_servers_joined: {{ data['site_info']['dns']['servers']}}
ingress_domain: {{ data['site_info']['domain']|lower }}
{% endfor %}
upstream_servers_joined: {{ data.site_info.dns.__str__() }}
node_domain: {{ data.site_info.domain | lower }}
ingress_domain: {{ data.site_info.domain | lower }}
genesis:
hostname: {{ (data|get_role_wise_nodes)['genesis']['name'] }}
{% for rack in data['baremetal'] %}
{% for host in data['baremetal'][rack] %}
{% if data['baremetal'][rack][host]['type'] == 'genesis' %}
ip: {{ data['baremetal'][rack][host]['ip']['calico'] }}
{% endif %}
{% endfor %}
{% endfor %}
hostname: {{ data.get_baremetal_host_by_type('genesis')[0].name }}
ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.calico }}
bootstrap:
ip: {{ (data|get_role_wise_nodes)['genesis']['pxe'] }}
ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.pxe }}
kubernetes:
api_service_ip: 10.96.0.1
etcd_service_ip: 10.96.0.2
pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16
# misc k8s port settings
apiserver_port: 6443
haproxy_port: 6553
service_node_port_range: 30000-32767
# etcd port settings
etcd:
container_port: 2379
haproxy_port: 2378
masters:
{% for host in (data|get_role_wise_nodes)['masters'] %}
- hostname: {{ host }}
{% endfor %}
# NEWSITE-CHANGEME: Environment proxy information.
# NOTE: Reference Airship sites do not deploy behind a proxy, so this proxy section
# should be commented out.
# However if you are in a lab that requires proxy, ensure that these proxy
# settings are correct and reachable in your environment; otherwise update
# them with the correct values for your environment.
proxy:
http: ""
https: ""
no_proxy: []
node_ports:
drydock_api: 30000
maas_api: 30001
maas_proxy: 31800 # hardcoded in MAAS
shipyard_api: 30003
airflow_web: 30004
ntp:
servers_joined: {{ data['site_info']['ntp']['servers'] }}
ldap:
base_url: {{ (data['site_info']['ldap']['url']|string).split('//')[1] }}
url: {{ data['site_info']['ldap']['url'] }}
auth_path: DC=test,DC=test,DC=com?sAMAccountName?sub?memberof=CN={{ data['site_info']['ldap']['common_name'] }},OU=Application,OU=Groups,DC=test,DC=test,DC=com
common_name: {{ data['site_info']['ldap']['common_name'] }}
subdomain: {{ data['site_info']['ldap']['subdomain'] }}
domain: {{ (data['site_info']['ldap']['url']|string).split('.')[1] }}
{% for host in data.get_baremetal_host_by_type('controller') %}
- hostname: {{ host.name }}
{% endfor %}
storage:
ceph:
public_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
cluster_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
public_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
cluster_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
neutron:
tunnel_device: 'bond1.{{ data['network']['vlan_network_data']['overlay']['vlan'] }}'
external_iface: 'bond1'
tunnel_device: "bond1.{{ data.network.get_vlan_data_by_name('overlay').vlan }}"
external_iface: "bond1"
openvswitch:
external_iface: 'bond1'
external_iface: "bond1"
...

216
spyglass/examples/templates/networks/physical/networks.yaml.j2
View File

@ -1,28 +1,4 @@
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: oob
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
# MaaS doesnt own this network like it does the others, so the noconfig label
# is specified.
labels:
noconfig: enabled
bonding:
mode: disabled
mtu: 1500
linkspeed: auto
trunking:
mode: disabled
default_network: oob
allowed_networks:
- oob
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
@ -30,42 +6,33 @@ metadata:
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: oob
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['oob']['subnet'] }}
cidr: {{ data.network.get_vlan_data_by_name('oob').subnet[0] }}
routes:
- subnet: '0.0.0.0/0'
gateway: {{ data['network']['vlan_network_data']['oob']['gateway'] }}
gateway: {{ data.network.get_vlan_data_by_name('oob').gateway }}
metric: 100
ranges:
- type: reserved
start: {{ data.network.get_vlan_data_by_name('oob').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('oob').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['oob']['static_start'] }}
end: {{ data['network']['vlan_network_data']['oob']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('oob').static_start }}
end: {{ data.network.get_vlan_data_by_name('oob').static_end }}
- type: dhcp
start: {{ data.network.get_vlan_data_by_name('oob').dhcp_start }}
end: {{ data.network.get_vlan_data_by_name('oob').dhcp_end }}
...
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: pxe
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
bonding:
mode: disabled
mtu: 1500
linkspeed: auto
trunking:
mode: disabled
default_network: pxe
allowed_networks:
- pxe
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: pxe
@ -80,56 +47,21 @@ metadata:
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['pxe']['subnet'] }}
cidr: {{ data.network.get_vlan_data_by_name('pxe').subnet[0] }}
routes:
{% for other_subnet in data['network']['vlan_network_data']['pxe']['routes'] %}
- subnet: {{ other_subnet }}
gateway: {{ data['network']['vlan_network_data']['pxe']['gateway'] }}
- subnet: '0.0.0.0/0'
gateway: {{ data.network.get_vlan_data_by_name('pxe').gateway }}
metric: 100
{% endfor %}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['pxe']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['pxe']['reserved_end'] }}
start: {{ data.network.get_vlan_data_by_name('pxe').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('pxe').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['pxe']['static_start'] }}
end: {{ data['network']['vlan_network_data']['pxe']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('pxe').static_start }}
end: {{ data.network.get_vlan_data_by_name('pxe').static_end }}
- type: dhcp
start: {{ data['network']['vlan_network_data']['pxe']['dhcp_start'] }}
end: {{ data['network']['vlan_network_data']['pxe']['dhcp_end'] }}
...
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: data
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
bonding:
mode: 802.3ad
hash: layer3+4
peer_rate: fast
mon_rate: 100
up_delay: 1000
down_delay: 3000
# NEWSITE-CHANGEME: Ensure the network switches in the environment are
# configured for this MTU or greater. Even if switches are configured for or
# can support a slightly higher MTU, there is no need (and negliable benefit)
# to squeeze every last byte into the MTU (e.g., 9216 vs 9100). Leave MTU at
# 9100 for maximum compatibility.
mtu: 9100
linkspeed: auto
trunking:
mode: 802.1q
allowed_networks:
- oam
- storage
- overlay
- calico
start: {{ data.network.get_vlan_data_by_name('pxe').dhcp_start }}
end: {{ data.network.get_vlan_data_by_name('pxe').dhcp_end }}
...
---
@ -148,27 +80,24 @@ metadata:
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['oam']['subnet'] }}
{% set flag = [0] %}
{% for route in data['network']['vlan_network_data']['oam']['routes'] %}
{% if flag[0] == 0 %}
cidr: {{ data.network.get_vlan_data_by_name('oam').subnet[0] }}
{% if data.network.get_vlan_data_by_name('oam').routes %}
routes:
{% endif %}
{% if flag.append(flag.pop() + 1) %} {% endif %}
{% for route in data.network.get_vlan_data_by_name('oam').routes %}
- subnet: {{ route }}
gateway: {{ data['network']['vlan_network_data']['oam']['gateway'] }}
gateway: {{ data.network.get_vlan_data_by_name('oam').gateway }}
metric: 100
{% endfor %}
{% if flag[0] == 0 %}
routes:[]
{% endif %}
{% endfor %}
{% else %}
routes: []
{% endif %}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['oam']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['oam']['reserved_end'] }}
start: {{ data.network.get_vlan_data_by_name('oam').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('oam').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['oam']['static_start'] }}
end: {{ data['network']['vlan_network_data']['oam']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('oam').static_start }}
end: {{ data.network.get_vlan_data_by_name('oam').static_end }}
...
---
@ -187,40 +116,14 @@ metadata:
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }}
cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['storage']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['storage']['reserved_end'] }}
start: {{ data.network.get_vlan_data_by_name('storage').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('storage').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['storage']['static_start'] }}
end: {{ data['network']['vlan_network_data']['storage']['static_end'] }}
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: overlay
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: os-overlay
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['overlay']['subnet'] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['overlay']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['overlay']['reserved_end'] }}
- type: static
start: {{ data['network']['vlan_network_data']['overlay']['static_start'] }}
end: {{ data['network']['vlan_network_data']['overlay']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('storage').static_start }}
end: {{ data.network.get_vlan_data_by_name('storage').static_end }}
...
---
@ -239,13 +142,38 @@ metadata:
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['calico']['subnet'] }}
cidr: {{ data.network.get_vlan_data_by_name('calico').subnet[0] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['calico']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['calico']['reserved_end'] }}
start: {{ data.network.get_vlan_data_by_name('calico').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('calico').reserved_end }}
- type: static
start: {{ data['network']['vlan_network_data']['calico']['static_start'] }}
end: {{ data['network']['vlan_network_data']['calico']['static_end'] }}
start: {{ data.network.get_vlan_data_by_name('calico').static_start }}
end: {{ data.network.get_vlan_data_by_name('calico').static_end }}
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: overlay
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: os-overlay
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data.network.get_vlan_data_by_name('overlay').subnet[0] }}
ranges:
- type: reserved
start: {{ data.network.get_vlan_data_by_name('overlay').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('overlay').reserved_end }}
- type: static
start: {{ data.network.get_vlan_data_by_name('overlay').static_start }}
end: {{ data.network.get_vlan_data_by_name('overlay').static_end }}
...

145
spyglass/examples/templates/pki/pki-catalogue.yaml.j2
View File

@ -21,33 +21,29 @@ data:
- 10.96.0.1
kubernetes_service_names:
- kubernetes.default.svc.cluster.local
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
{% for host in data.get_baremetal_host_by_type('genesis') %}
- document_name: kubelet-genesis
common_name: system:node:{{ host }}
common_name: system:node:{{ host.name }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
groups:
- system:nodes
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
- document_name: kubelet-{{ host }}
common_name: system:node:{{ host }}
{% endfor %}
{% for rack in data.baremetal %}
{% for host in rack.hosts %}
- document_name: kubelet-{{ host.name }}
common_name: system:node:{{ host.name }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
groups:
- system:nodes
{%endfor%}
{%endfor%}
{% endfor %}
{% endfor %}
- document_name: scheduler
description: Service certificate for Kubernetes scheduler
common_name: system:kube-scheduler
@ -62,6 +58,7 @@ data:
common_name: armada
groups:
- system:masters
kubernetes-etcd:
description: Certificates for Kubernetes's etcd servers
certificates:
@ -72,113 +69,93 @@ data:
- document_name: kubernetes-etcd-anchor
description: anchor
common_name: anchor
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
{% for host in data.get_baremetal_host_by_type('genesis') %}
- document_name: kubernetes-etcd-genesis
common_name: kubernetes-etcd-genesis
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis'%}
- document_name: kubernetes-etcd-{{ host }}
common_name: kubernetes-etcd-{{ host }}
{% endfor %}
{% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
- document_name: kubernetes-etcd-{{ host.name }}
common_name: kubernetes-etcd-{{ host.name }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
{% endfor %}
kubernetes-etcd-peer:
certificates:
{% for host in data.get_baremetal_host_by_type('genesis') %}
- document_name: kubernetes-etcd-genesis-peer
common_name: kubernetes-etcd-genesis-peer
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: kubernetes-etcd-{{ host }}-peer
common_name: kubernetes-etcd-{{ host }}-peer
{% endfor %}
{% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
- document_name: kubernetes-etcd-{{ host.name }}-peer
common_name: kubernetes-etcd-{{ host.name }}-peer
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2
{% endif %}
{%endfor%}
{%endfor%}
ksn-etcd:
{% endfor %}
calico-etcd:
description: Certificates for Calico etcd client traffic
certificates:
- document_name: ksn-etcd-anchor
description: anchor
common_name: anchor
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: ksn-etcd-{{ host }}
common_name: ksn-etcd-{{ host }}
{% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
- document_name: ksn-etcd-{{ host.name }}
common_name: ksn-etcd-{{ host.name }}
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- 10.96.232.136
{% endif %}
{%endfor%}
{%endfor%}
{% endfor %}
- document_name: ksn-node
common_name: calcico-node
ksn-etcd-peer:
calico-etcd-peer:
description: Certificates for Calico etcd clients
certificates:
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: ksn-etcd-{{ host }}-peer
common_name: ksn-etcd-{{ host }}-peer
{% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
- document_name: ksn-etcd-{{ host.name }}-peer
common_name: ksn-etcd-{{ host.name }}-peer
hosts:
- {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['calico']}}
- {{ host.name }}
- {{ host.ip.oam }}
- {{ host.ip.calico }}
- 127.0.0.1
- localhost
- 10.96.232.136
{% endif %}
{%endfor%}
{%endfor%}
{% endfor %}
- document_name: ksn-node-peer
common_name: calico-node-peer
keypairs:

4
spyglass/examples/templates/profile/region.yaml.j2
View File

@ -2,7 +2,7 @@
schema: 'drydock/Region/v1'
metadata:
schema: 'metadata/Document/v1'
name: {{ data['region_name'] }}
name: {{ data.site_info.region_name }}
layeringDefinition:
abstract: false
layer: site
@ -18,7 +18,7 @@ metadata:
path: .authorized_keys[1]
src:
schema: deckhand/PublicKey/v1
name: {{ data['region_name'] }}_ssh_public_key
name: {{ data.site_info.region_name }}_ssh_public_key
path: .
- dest:
path: .repositories.main_archive

4
spyglass/examples/templates/site-definition.yaml.j2
View File

@ -7,11 +7,11 @@ metadata:
abstract: false
layer: site
# NEWSITE-CHANGEME: Replace with the site name
name: {{ data['region_name'] }}
name: {{ data.site_info.region_name }}
storagePolicy: cleartext
data:
# The type layer this site will delpoy with. Type layer is found in the
# aic-clcp-manifests repo.
site_type: {{ data['site_info']['sitetype'] }}
site_type: {{ data.site_info.sitetype }}
...

55
spyglass/examples/templates/software/charts/kubernetes/etcd.yaml.j2
View File

@ -13,83 +13,70 @@ metadata:
path: .
storagePolicy: cleartext
substitutions:
{% set count = [0] %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'controller' %}
{% for host in data.get_baremetal_host_by_type('controller') %}
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .masters[{{ count[0] }}].hostname
path: .masters[{{ loop.index - 1 }}].hostname
dest:
path: .values.nodes[{{ count[0] }}].name
path: .values.nodes[{{ loop.index - 1 }}].name
- src:
schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.key
path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src:
schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}-peer
name: calico-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key
{% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #}
{% endif %}
{% endfor %}
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endfor %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'genesis' %}
{% for host in data.get_baremetal_host_by_type('genesis') %}
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .genesis.hostname
dest:
path: .values.nodes[{{ count[0] }}].name
path: .values.nodes[{{ loop.index - 1 }}].name
- src:
schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}
name: calico-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.key
path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src:
schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }}-peer
name: calico-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src:
schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}-peer
name: calico-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key
{% endif %}
{% endfor %}
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endfor %}
data: {}

48
spyglass/examples/templates/software/charts/kubernetes/etcd/etcd.yaml.j2
View File

@ -13,80 +13,70 @@ metadata:
path: .
storagePolicy: cleartext
substitutions:
{% set count = [0] %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'controller' %}
{% for host in data.get_baremetal_host_by_type('controller') %}
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .masters[{{ count[0] }}].hostname
path: .masters[{{ loop.index - 1 }}].hostname
dest:
path: .values.nodes[{{ count[0] }}].name
path: .values.nodes[{{ loop.index - 1 }}].name
- src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-{{ host }}
name: kubernetes-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-{{ host }}
name: kubernetes-etcd-{{ host.name }}
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.key
path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-{{ host }}-peer
name: kubernetes-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-{{ host }}-peer
name: kubernetes-etcd-{{ host.name }}-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key
{% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #}
{% endif %}
{% endfor %}
{% endfor %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'genesis' %}
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endfor %}
{% for host in data.get_baremetal_host_by_type('genesis') %}
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .genesis.hostname
dest:
path: .values.nodes[{{ count[0] }}].name
path: .values.nodes[{{ loop.index - 1 }}].name
- src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-genesis
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-genesis
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.client.key
path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src:
schema: deckhand/Certificate/v1
name: kubernetes-etcd-genesis-peer
path: .
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src:
schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-genesis-peer
path: $
dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key
{% endif %}
{% endfor %}
{% endfor %}
path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endfor %}
data: {}
...

2
spyglass/examples/templates/software/charts/osh/openstack-tenant-ceph/ceph-client.yaml.j2
View File

@ -18,5 +18,5 @@ data:
conf:
pool:
target:
osd: {{ data['storage']['ceph']['controller']['osd_count'] }}
osd: {{ data.storage['ceph']['controller']['osd_count'] }}
...

3
spyglass/examples/templates/software/config/common-software-config.yaml.j2
View File

@ -13,4 +13,5 @@ metadata:
data:
osh:
# NEWSITE-CHANGEME: Replace with the site name
region_name: {{ data['region_name'] }}
region_name: {{ data.site_info.region_name }}
...

19
spyglass/site_processors/base.py
View File

@ -20,22 +20,3 @@ class BaseProcessor(object):
def render_template(self, template):
pass
@staticmethod
def get_role_wise_nodes(yaml_data):
hosts = {"genesis": {}, "masters": [], "workers": []}
for rack in yaml_data["baremetal"]:
for host in yaml_data["baremetal"][rack]:
if yaml_data["baremetal"][rack][host]["type"] == "genesis":
hosts["genesis"] = {
"name": host,
"pxe": yaml_data["baremetal"][rack][host]["ip"]["pxe"],
"oam": yaml_data["baremetal"][rack][host]["ip"]["oam"],
}
elif yaml_data["baremetal"][rack][host]["type"] \
== "controller":
hosts["masters"].append(host)
else:
hosts["workers"].append(host)
return hosts

19
spyglass/site_processors/site_processor.py
View File

@ -18,6 +18,8 @@ import shutil
import jinja2
from spyglass.data_extractor.models import site_document_data_factory
from spyglass.data_extractor.models import SiteDocumentData
from spyglass.site_processors.base import BaseProcessor
LOG = logging.getLogger(__name__)
@ -25,9 +27,12 @@ LOG = logging.getLogger(__name__)
class SiteProcessor(BaseProcessor):
def __init__(self, intermediary_yaml, manifest_dir, force_write):
def __init__(self, site_data, manifest_dir, force_write):
super().__init__()
self.yaml_data = intermediary_yaml
if isinstance(site_data, SiteDocumentData):
self.site_data = site_data
else:
self.site_data = site_document_data_factory(site_data)
self.manifest_dir = manifest_dir
self.force_write = force_write
@ -38,7 +43,7 @@ class SiteProcessor(BaseProcessor):
calico) are generated in a single file. Rack specific
configs( pxe and oob) are generated per rack.
"""
# Check of manifest_dir exists
# Check if manifest_dir exists
if self.manifest_dir is not None:
site_manifest_dir = os.path.join(
self.manifest_dir, 'pegleg_manifests', 'site')
@ -66,16 +71,16 @@ class SiteProcessor(BaseProcessor):
autoescape=True,
loader=loader,
trim_blocks=True,
lstrip_blocks=True,
undefined=logging_undefined)
j2_env.filters["get_role_wise_nodes"] = \
self.get_role_wise_nodes
templatefile = os.path.join(dirpath, filename)
LOG.debug("Template file: %s", templatefile)
outdirs = dirpath.split(template_folder_name)[1].lstrip(os.sep)
LOG.debug("outdirs: %s", outdirs)
outfile_path = os.path.join(
site_manifest_dir, self.yaml_data["region_name"], outdirs)
site_manifest_dir, self.site_data.site_info.region_name,
outdirs)
LOG.debug("outfile path: %s", outfile_path)
outfile_yaml = os.path.split(templatefile)[1]
outfile_yaml = os.path.splitext(outfile_yaml)[0]
@ -90,7 +95,7 @@ class SiteProcessor(BaseProcessor):
out = open(outfile, "w")
created_file_list.append(outfile)
LOG.info("Rendering {}".format(outfile_yaml))
rendered = template_j2.render(data=self.yaml_data)
rendered = template_j2.render(data=self.site_data)
out.write(rendered)
out.close()
except IOError as ioe:

187
tests/unit/data_extractor/test_models.py
View File

@ -13,11 +13,18 @@
# limitations under the License.
from copy import copy
import os
import unittest
from unittest import mock
import yaml
from spyglass.data_extractor.custom_exceptions import InvalidIntermediary
from spyglass.data_extractor import models
FIXTURE_DIR = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'shared')
class TestParseIp(unittest.TestCase):
"""Tests the _parse_ip validator for Spyglass models"""
@ -340,11 +347,7 @@ class TestRack(unittest.TestCase):
"""Tests for the Rack model"""
RACK_NAME = 'test_rack1'
HOST_DATA = {
'rack_name': RACK_NAME,
'host_profile': 'host',
'type': 'compute'
}
HOST_DATA = {'rack_name': RACK_NAME, 'host_profile': 'host'}
@mock.patch('spyglass.data_extractor.models.IPList', autospec=True)
def setUp(self, MockIPList):
@ -353,9 +356,9 @@ class TestRack(unittest.TestCase):
self.HOST_DATA['ip'] = MockIPList()
self.HOST_DATA['ip'].dict_from_class.return_value = 'success'
self.hosts = [
models.Host('test_host1', **self.HOST_DATA),
models.Host('test_host2', **self.HOST_DATA),
models.Host('test_host3', **self.HOST_DATA),
models.Host('test_host1', **self.HOST_DATA, type='genesis'),
models.Host('test_host2', **self.HOST_DATA, type='compute'),
models.Host('test_host3', **self.HOST_DATA, type='controller'),
]
def test___init__(self):
@ -408,6 +411,15 @@ class TestRack(unittest.TestCase):
self.assertEqual(
self.hosts[1], result.get_host_by_name(self.hosts[1].name))
def test_get_host_by_type(self):
"""Tests retrieval of a Rack's host(s) by type"""
result = models.Rack(self.RACK_NAME, self.hosts)
self.assertEqual(self.hosts[0], result.get_host_by_type('genesis')[0])
self.assertEqual(self.hosts[1], result.get_host_by_type('compute')[0])
self.assertEqual(
self.hosts[2],
result.get_host_by_type('controller')[0])
class TestVLANNetworkData(unittest.TestCase):
"""Tests for the VLANNetworkData model"""
@ -820,3 +832,162 @@ class TestSiteDocumentData(unittest.TestCase):
type(Rack()).name = mock.PropertyMock(side_effect=['rack1', 'rack3'])
result = models.SiteDocumentData(site_info, network, baremetal)
self.assertIsNone(result.get_baremetal_rack_by_name('rack2'))
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.Network')
@mock.patch('spyglass.data_extractor.models.Rack')
@mock.patch('spyglass.data_extractor.models.Host')
def test_get_baremetal_rack_by_name_multiple(
self, Host, Rack, Network, SiteInfo):
"""Tests retrieval of baremetal host(s) by type"""
site_info = SiteInfo()
network = Network()
baremetal = [Rack(), Rack()]
Rack().get_host_by_type.return_value = [Host()]
result = models.SiteDocumentData(site_info, network, baremetal)
self.assertEqual(2, len(result.get_baremetal_host_by_type('genesis')))
self.assertEqual(2, len(result.get_baremetal_host_by_type('computer')))
self.assertEqual(
2, len(result.get_baremetal_host_by_type('controller')))
class TestValidateKeyInIntermediaryDict(unittest.TestCase):
"""Tests the _validate_key_in_intermediary_dict function"""
def test__validate_key_in_intermediary_dict(self):
test_dictionary = {'test_key': 'value'}
key = 'test_key'
self.assertIsNone(
models._validate_key_in_intermediary_dict(key, test_dictionary))
def test__validate_key_in_intermediary_dict_key_dne(self):
test_dictionary = {'test_key': 'value'}
key = 'not_test_key'
with self.assertRaises(InvalidIntermediary):
models._validate_key_in_intermediary_dict(key, test_dictionary)
class TestSiteDocumentDataFactory(unittest.TestCase):
"""Tests the site_document_data_factory function"""
def setUp(self) -> None:
test_intermediary_path = os.path.join(
FIXTURE_DIR, 'test_intermediary.yaml')
with open(test_intermediary_path, 'r') as f:
self.intermediary_dict = yaml.safe_load(f)
def test_site_document_data_factory(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check correct return type
self.assertIsInstance(site_document_data, models.SiteDocumentData)
def test_site_document_data_factory_saves_storage(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that storage was saved without changes in the SiteDocumentData
self.assertDictEqual(
self.intermediary_dict['storage'], site_document_data.storage)
def test_site_document_data_factory_saves_site_info(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that site info saved correctly in SiteInfo object
site_info_dict = self.intermediary_dict['site_info']
self.assertIsInstance(site_document_data.site_info, models.SiteInfo)
self.assertEqual(
site_info_dict['name'], site_document_data.site_info.name)
self.assertEqual(
self.intermediary_dict['region_name'],
site_document_data.site_info.region_name)
self.assertEqual(
site_info_dict['state'], site_document_data.site_info.state)
self.assertEqual(
site_info_dict['physical_location_id'],
site_document_data.site_info.physical_location_id)
self.assertEqual(
site_info_dict['country'], site_document_data.site_info.country)
self.assertEqual(
site_info_dict['corridor'], site_document_data.site_info.corridor)
self.assertEqual(
site_info_dict['sitetype'], site_document_data.site_info.sitetype)
self.assertEqual(
site_info_dict['domain'], site_document_data.site_info.domain)
self.assertDictEqual(
site_info_dict['ldap'], site_document_data.site_info.ldap)
self.assertEqual(
site_info_dict['dns']['servers'],
str(site_document_data.site_info.dns))
self.assertEqual(
site_info_dict['ntp']['servers'],
str(site_document_data.site_info.ntp))
def test_site_document_data_factory_saves_network_data(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that network data saved correctly into a Network object
network_dict = self.intermediary_dict['network']
self.assertIsInstance(site_document_data.network, models.Network)
self.assertDictEqual(
network_dict['bgp'], site_document_data.network.bgp)
for network_type, network_data \
in network_dict['vlan_network_data'].items():
vlan_network_data = \
site_document_data.network.get_vlan_data_by_name(network_type)
self.assertIsInstance(vlan_network_data, models.VLANNetworkData)
self.assertEqual(network_type, vlan_network_data.name)
self.assertEqual(network_type, vlan_network_data.role)
self.assertEqual(network_data['subnet'], vlan_network_data.subnet)
if 'routes' in network_data:
self.assertEqual(
network_data['routes'], vlan_network_data.routes)
if 'gateway' in network_data:
self.assertEqual(
network_data['gateway'], vlan_network_data.gateway)
if 'vlan' in network_data:
self.assertEqual(network_data['vlan'], vlan_network_data.vlan)
if 'dhcp_start' in network_data and 'dhcp_end' in network_data:
self.assertEqual(
network_data['dhcp_start'], vlan_network_data.dhcp_start)
self.assertEqual(
network_data['dhcp_end'], vlan_network_data.dhcp_end)
if 'static_start' in network_data and 'static_end' in network_data:
self.assertEqual(
network_data['static_start'],
vlan_network_data.static_start)
self.assertEqual(
network_data['static_end'], vlan_network_data.static_end)
if 'reserved_start' in network_data \
and 'reserved_end' in network_data:
self.assertEqual(
network_data['reserved_start'],
vlan_network_data.reserved_start)
self.assertEqual(
network_data['reserved_end'],
vlan_network_data.reserved_end)
def test_site_document_data_factory_saves_baremetal_data(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that baremetal racks saved correctly into Rack objects
for rack_name, hosts \
in self.intermediary_dict['baremetal'].items():
rack = site_document_data.get_baremetal_rack_by_name(rack_name)
for host_name, host_data in hosts.items():
host = rack.get_host_by_name(host_name)
self.assertEqual(host_name, host.name)
self.assertEqual(rack_name, host.rack_name)
self.assertEqual(host_data['type'], host.type)
self.assertEqual(host_data['host_profile'], host.host_profile)
self.assertEqual(host_data['ip']['oob'], host.ip.oob)
self.assertEqual(host_data['ip']['oam'], host.ip.oam)
self.assertEqual(host_data['ip']['calico'], host.ip.calico)
self.assertEqual(host_data['ip']['overlay'], host.ip.overlay)
self.assertEqual(host_data['ip']['pxe'], host.ip.pxe)
self.assertEqual(host_data['ip']['storage'], host.ip.storage)
self.assertEqual(rack_name, rack.name)

247
tests/unit/site_processors/test_site_processor.py
View File

@ -15,117 +15,178 @@
import logging
import os
from tempfile import mkdtemp
import textwrap
import unittest
from unittest import mock
from jinja2 import UndefinedError
import pytest
from spyglass.data_extractor import models
from spyglass.site_processors.site_processor import SiteProcessor
LOG = logging.getLogger(__name__)
LOG.level = logging.DEBUG
J2_TPL = """---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: {{ data['region_name'] }}
storagePolicy: cleartext
data:
site_type:{{ data['site_info']['sitetype'] }}
..."""
class TestSiteProcessor(unittest.TestCase):
def test_render_template():
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
_input_yaml = {
"region_name": "test",
"site_info": {
"sitetype": "test_type"
}
}
_out_dir = mkdtemp()
site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False)
site_processor.render_template(_tpl_parent_dir)
J2_TPL = textwrap.dedent(
"""
---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: {{ data.site_info.region_name }}
storagePolicy: cleartext
data:
site_type:{{ data.site_info.sitetype }}
...""")
expected_output = """---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:test_type
..."""
J2_TPL_UNDEFINED = textwrap.dedent(
"""
---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: {{ data.site_info.region_name }}
storagePolicy: cleartext
data:
site_type:{{ undefined_param }}
...""")
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", _input_yaml["region_name"],
os.path.split(_tpl_dir)[1], "test.yaml")
LOG.debug(output_file)
assert (os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
assert (expected_output == content)
@mock.patch(
'spyglass.data_extractor.models.SiteDocumentData',
spec=models.SiteDocumentData)
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.ServerList')
def test_render_template(self, ServerList, SiteInfo, SiteDocumentData):
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(self.J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
site_data = SiteDocumentData()
type(SiteDocumentData()).site_info = SiteInfo()
region_name = 'test'
type(SiteInfo()).region_name = mock.PropertyMock(
return_value=region_name)
site_type = 'test_type'
type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
def test_render_template_missing_data():
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
_input_yaml = {"region_name": "test", "site_info": {}}
_out_dir = mkdtemp()
site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False)
with pytest.raises(UndefinedError):
_out_dir = mkdtemp()
site_processor = SiteProcessor(site_data, _out_dir, force_write=False)
site_processor.render_template(_tpl_parent_dir)
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", _input_yaml["region_name"],
os.path.split(_tpl_dir)[1], "test.yaml")
assert (not os.path.exists(output_file))
expected_output = textwrap.dedent(
"""
---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:test_type
...""")
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", region_name,
os.path.split(_tpl_dir)[1], "test.yaml")
LOG.debug(output_file)
self.assertTrue(os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
self.assertEqual(expected_output, content)
def test_render_template_missing_data_force():
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
_input_yaml = {"region_name": "test", "site_info": {}}
_out_dir = mkdtemp()
site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=True)
site_processor.render_template(_tpl_parent_dir)
@mock.patch(
'spyglass.data_extractor.models.SiteDocumentData',
spec=models.SiteDocumentData)
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.ServerList')
def test_render_template_missing_data(
self, ServerList, SiteInfo, SiteDocumentData):
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(self.J2_TPL_UNDEFINED)
LOG.debug("Writing test template to %s", _tpl_file)
expected_output = """---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:
..."""
site_data = SiteDocumentData()
type(SiteDocumentData()).site_info = SiteInfo()
region_name = 'test'
type(SiteInfo()).region_name = mock.PropertyMock(
return_value=region_name)
site_type = 'test_type'
type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", _input_yaml["region_name"],
os.path.split(_tpl_dir)[1], "test.yaml")
assert (os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
assert (expected_output == content)
_out_dir = mkdtemp()
site_processor = SiteProcessor(site_data, _out_dir, force_write=False)
with pytest.raises(UndefinedError):
site_processor.render_template(_tpl_parent_dir)
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", region_name,
os.path.split(_tpl_dir)[1], "test.yaml")
self.assertFalse(os.path.exists(output_file))
@mock.patch(
'spyglass.data_extractor.models.SiteDocumentData',
spec=models.SiteDocumentData)
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.ServerList')
def test_render_template_missing_data_force(
self, ServerList, SiteInfo, SiteDocumentData):
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(self.J2_TPL_UNDEFINED)
LOG.debug("Writing test template to %s", _tpl_file)
site_data = SiteDocumentData()
type(SiteDocumentData()).site_info = SiteInfo()
region_name = 'test'
type(SiteInfo()).region_name = mock.PropertyMock(
return_value=region_name)
site_type = 'test_type'
type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
_out_dir = mkdtemp()
site_processor = SiteProcessor(site_data, _out_dir, force_write=True)
site_processor.render_template(_tpl_parent_dir)
expected_output = textwrap.dedent(
"""
---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:
...""")
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", region_name,
os.path.split(_tpl_dir)[1], "test.yaml")
self.assertTrue(os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
self.assertEqual(expected_output, content)