airship
/
drydock
Code Issues Proposed changes

Initial Python skeleton and the model 

classes to match the YAML schema
This commit is contained in:
Scott Hussey 2017-02-22 17:32:55 -06:00
parent 04d8a5e9d9
commit 07cb34e82d
18 changed files with 705 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
helm_drydock/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

35
helm_drydock/config.py Normal file
View File

@ -0,0 +1,35 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Read application configuration
#
# configuration map with defaults
class DrydockConfig(object):
def __init__(self):
self.selected_server_driver = helm_drydock.drivers.server.maasdriver
self.selected_network_driver = helm_drydock.drivers.network.noopdriver
self.control_config = {}
self.ingester_config = {
plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]
}
self.introspection_config = {}
self.orchestrator_config = {}
self.statemgmt_config = {
backend_driver = 'helm_drydock.drivers.statemgmt.etcd',
}

19
helm_drydock/drivers/__init__.py Normal file
View File

@ -0,0 +1,19 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class ProviderDriver(object):
__init__(self):
pass

18
helm_drydock/drivers/server/__init__.py Normal file
View File

@ -0,0 +1,18 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.drivers.ProviderDriver
class ServerDriver(ProviderDriver):

17
helm_drydock/drivers/server/maasdriver/__init__.py Normal file
View File

@ -0,0 +1,17 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.drivers.server.ServerDriver
class MaasServerDriver(object):

72
helm_drydock/ingester/ingester.py Normal file
View File

@ -0,0 +1,72 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ingester - Ingest host topologies to define site design and
# persist design to helm-drydock's statemgmt service
import logging
import yaml
class Ingester(object):
registered_plugins = {}
def __init__(self):
logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s")
self.log = logging.Logger("ingester")
"""
enable_plugins
params: plugins - A list of class objects denoting the ingester plugins to be enabled
Enable plugins that can be used for ingest_data calls. Each plugin should use
helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one
enabled plugin successfully initializes, the call is considered successful. Otherwise
it will throw an exception
"""
def enable_plugins(self, plugins=[]):
if len(plugin) == 0:
self.log.error("Cannot have an empty plugin list.")
for plugin in plugins:
try:
new_plugin = plugin()
plugin_name = new_plugin.get_name()
registered_plugins[plugin_name] = new_plugin
except:
self.log.error("Could not enable plugin %s" % (plugin.__name__))
if len(registered_plugins) == 0:
self.log.error("Could not enable at least one plugin")
raise Exception("Could not enable at least one plugin")
"""
ingest_data
params: plugin_name - Which plugin should be used for ingestion
params: params - A map of parameters that will be passed to the plugin's ingest_data method
Execute a data ingestion using the named plugin (assuming it is enabled)
"""
def ingest_data(self, plugin_name, params={}):
if plugin_name in registered_plugins:
design_data = registered_plugins[plugin_name].ingest_data(params)
# Need to persist data here, but we don't yet have the statemgmt service working
yaml.dump(design_data)
else
self.log.error("Could not find plugin %s to ingest data." % (plugin_name))
raise LookupError("Could not find plugin %s" % plugin_name)

55
helm_drydock/ingester/plugins/aicyaml.py Normal file
View File

@ -0,0 +1,55 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# AIC YAML Ingester - This data ingester will consume a AIC YAML design
# file
#
import yaml
import logging
import helm_drydock.ingester.plugins.IngesterPlugin
class AicYamlIngester(IngesterPlugin):
def __init__(self):
super(AicYamlIngester, self).__init__()
def get_name(self):
return "aic_yaml"
"""
AIC YAML ingester params
filename - Absolute path to the YAML file to ingest
"""
def ingest_data(self, **kwargs):
if 'filename' in params:
input_string = read_input_file(params['filename'])
parsed_data = parse_input_data(input_string)
processed_data = compute_effective_data(parsed_data)
else:
raise Exception('Missing parameter')
return processed_data
def read_input_file(self, filename):
try:
file = open(filename,'rt')
except OSError as err:
self.log.error("Error opening input file %s for ingestion: %s" % (filename, err))
return {}

30
helm_drydock/ingester/plugins/ingester_plugin.py Normal file
View File

@ -0,0 +1,30 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Plugins to parse incoming topology and translate it to helm-drydock's
# intermediate representation
import logging
class IngesterPlugin(object):
def __init__(self):
self.log = logging.Logger('ingester')
return
def get_data(self):
return "ingester_skeleton"
def ingest_data(self, **kwargs):
return {}

286
helm_drydock/model/__init__.py Normal file
View File

@ -0,0 +1,286 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Models for helm_drydock
#
class HardwareProfile(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
self.region = metadata.get('region', '')
self.vendor = spec.get('vendor', '')
self.generation = spec.get('generation', '')
self.hw_version = spec.get('hw_version', '')
self.bios_version = spec.get('bios_version', '')
self.boot_mode = spec.get('boot_mode', '')
self.bootstrap_protocol = spec.get('bootstrap_protocol', '')
self.pxe_interface = spec.get('pxe_interface', '')
self.devices = []
device_aliases = spec.get('device_aliases', {})
pci_devices = device_aliases.get('pci', [])
scsi_devices = device_aliases.get('scsi', [])
for d in pci_devices:
d['bus_type'] = 'pci'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
for d in scsi_devices:
d['bus_type'] = 'scsi'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
else:
raise ValueError('Unknown API version of object')
return
class HardwareDeviceAlias(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "1.0":
self.bus_type = kwargs.get('bus_type', '')
self.address = kwargs.get('address', '')
self.alias = kwargs.get('alias', '')
self.type = kwargs.get('type', '')
else:
raise ValueError('Unknown API version of object')
return
class Site(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "1.0":
metadata = kwargs.get('metadata', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
self.networks = []
self.network_links = []
self.host_profiles = []
self.hardware_profiles = []
self.baremetal_nodes = []
else:
raise ValueError('Unknown API version of object')
class NetworkLink(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.region = metadata.get('region', '')
bonding = spec.get('bonding', {})
self.bonding_mode = bonding.get('mode', 'none')
# TODO How should we define defaults for CIs not in the input?
if self.bonding_mode == '802.3ad':
self.bonding_xmit_hash = bonding.get('hash', 'layer3+4')
self.bonding_peer_rate = bonding.get('peer_rate', 'fast')
self.bonding_mon_rate = bonding.get('mon_rate', '')
self.bonding_up_delay = bonding.get('up_delay', '')
self.bonding_down_delay = bonding.get('down_delay', '')
self.mtu = spec.get('mtu', 1500)
self.linkspeed = spec.get('linkspeed', 'auto')
trunking = spec.get('trunking', {})
self.trunk_mode = trunking.get('mode', 'none')
self.native_network = spec.get('default_network', '')
else:
raise ValueError('Unknown API version of object')
class Network(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.region = metadata.get('region', '')
self.cidr = spec.get('cidr', '')
self.allocation_strategy = spec.get('allocation', 'static')
self.vlan_id = spec.get('vlan_id', 1)
self.mtu = spec.get('mtu', 0)
dns = spec.get('dns', {})
self.dns_domain = dns.get('domain', 'local')
self.dns_servers = dns.get('servers', '')
ranges = spec.get('ranges', [])
self.ranges = []
for r in ranges:
self.ranges.append(NetworkAddressRange(self.api_version, **r))
routes = spec.get('routes', [])
self.routes = []
for r in routes:
self.routes.append(NetworkRoute(self.api_version, **r))
else:
raise ValueError('Unknown API version of object')
class NetworkAddressRange(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "1.0":
self.type = kwargs.get('type', 'static')
self.start = kwargs.get('start', '')
self.end = kwargs.get('end', '')
else:
raise ValueError('Unknown API version of object')
class NetworkRoute(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "1.0":
self.type = kwargs.get('subnet', '')
self.start = kwargs.get('gateway', '')
self.end = kwargs.get('metric', 100)
else:
raise ValueError('Unknown API version of object')
class HostProfile(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.region = metadata.get('region', '')
oob = spec.get('oob', {})
self.oob_type = oob.get('type', 'ipmi')
self.oob_network = oob.get('network', 'oob')
self.oob_account = oob.get('account', '')
self.oob_credential = oob.get('credential', '')
storage = spec.get('storage', {})
self.storage_layout = storage.get('layout', 'lvm')
bootdisk = storage.get('bootdisk', {})
self.bootdisk_device = bootdisk.get('device', '')
self.bootdisk_root_size = bootdisk.get('root_size', '')
self.bootdisk_boot_size = bootdisk.get('boot_size', '')
partitions = storage.get('partitions', [])
self.partitions = []
for p in partitions:
self.partitions.append(HostPartition(self.api_version, **p))
interfaces = spec.get('interfaces', [])
self.interfaces = []
for i in interfaces:
self.interfaces.append(HostInterface(self.api_version, **i))
else:
raise ValueError('Unknown API version of object')
class HostInterface(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "1.0":
self.device_name = kwargs.get('device_name', '')
self.network_link = kwargs.get('device_link', '')
self.hardware_slaves = []
slaves = kwargs.get('slaves', [])
for s in slaves:
self.hardware_slaves.append(s)
self.networks = []
networks = kwargs.get('networks', [])
for n in networks:
self.networks.append(n)
else:
raise ValueError('Unknown API version of object')
class HostPartition(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "1.0":
self.name = kwargs.get('name', '')
self.device = kwargs.get('device', '')
self.part_uuid = kwargs.get('part_uuid', '')
self.size = kwargs.get('size', '')
self.mountpoint = kwargs.get('mountpoint', '')
self.fstype = kwargs.get('fstype', 'ext4')
self.mount_options = kwargs.get('mount_options', 'defaults')
self.fs_uuid = kwargs.get('fs_uuid', '')
self.fs_label = kwargs.get('fs_label', '')
else:
raise ValueError('Unknown API version of object')
# A BaremetalNode is really nothing more than a physical
# instantiation of a HostProfile, so they both represent
# the same set of CIs
class BaremetalNode(HostProfile):
def __init__(self, **kwargs):
super(BaremetalNode, self).__init__()

BIN
helm_drydock/model/__init__.pyc Normal file
View File

Binary file not shown.

11
helm_drydock/tox.ini Normal file
View File

@ -0,0 +1,11 @@
[tox]
envlist = py35
[testenv]
deps=
-rrequirements.txt
setenv=
PYTHONWARNING=all
[flake8]
ignore=E302,H306

7
requirements.txt Normal file
View File

@ -0,0 +1,7 @@
PyYAML
oauth
requests-oauthlib
pyipmi
netaddr
pecan
python-libmaas

56
setup.py Normal file
View File

@ -0,0 +1,56 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# helm_drydock - A tool to consume a host topology and orchestrate
# and monitor the provisioning of those hosts and execution of bootstrap
# scripts
#
# Modular services:
# smelter - A service to consume the host topology, will support multiple
# input formats. Initially supports a YAML schema as demonstrated
# in the examples folder
# tarot - A service for persisting the host topology and orchestration state
# and making the data available via API
# cockpit - The entrypoint API for users to control helm-drydock and query
# current state
# alchemist - The core orchestrator
# drivers - A tree with all of the plugins that alchemist uses to execute
# orchestrated tasks
# jabberwocky - An introspection API that newly provisioned nodes can use to
# ingest self-data and bootstrap their application deployment process
from setuptools import setup
setup(name='helm_drydock',
version='0.1a1',
description='Bootstrapper for Kubernetes infrastructure',
url='http://github.com/att-comdev/drydock',
author='Scott Hussey - AT&T',
author_email='sh8121@att.com',
license='Apache 2.0',
packages=['helm_drydock',
'helm_drydock.model',
'helm_drydock.ingester'],
install_requires=[
'PyYAML',
'oauth',
'requests-oauthlib',
'pyipmi',
'netaddr',
'pecan'
],
dependency_link=[
'git+https://github.com/maas/python-libmaas.git'
]
)

2
testrequirements.txt Normal file
View File

@ -0,0 +1,2 @@
pytest
tox

BIN
tests/__pycache__/test_models.cpython-35-PYTEST.pyc Normal file
View File

Binary file not shown.

69
tests/test_models.py Normal file
View File

@ -0,0 +1,69 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
import yaml
from helm_drydock.model import HardwareProfile
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_hardwareprofile(self):
yaml_snippet = ("---\n"
"apiVersion: '1.0'\n"
"kind: HardwareProfile\n"
"metadata:\n"
" name: HPGen8v3\n"
" region: sitename\n"
" date: 17-FEB-2017\n"
" name: Sample hardware definition\n"
" author: Scott Hussey\n"
"spec:\n"
" # Vendor of the server chassis\n"
" vendor: HP\n"
" # Generation of the chassis model\n"
" generation: '8'\n"
" # Version of the chassis model within its generation - not version of the hardware definition\n"
" hw_version: '3'\n"
" # The certified version of the chassis BIOS\n"
" bios_version: '2.2.3'\n"
" # Mode of the default boot of hardware - bios, uefi\n"
" boot_mode: bios\n"
" # Protocol of boot of the hardware - pxe, usb, hdd\n"
" bootstrap_protocol: pxe\n"
" # Which interface to use for network booting within the OOB manager, not OS device\n"
" pxe_interface: 0\n"
" # Map hardware addresses to aliases/roles to allow a mix of hardware configs\n"
" # in a site to result in a consistent configuration\n"
" device_aliases:\n"
" pci:\n"
" - address: pci@0000:00:03.0\n"
" alias: prim_nic01\n"
" # type could identify expected hardware - used for hardware manifest validation\n"
" type: '82540EM Gigabit Ethernet Controller'\n"
" - address: pci@0000:00:04.0\n"
" alias: prim_nic02\n"
" type: '82540EM Gigabit Ethernet Controller'\n"
" scsi:\n"
" - address: scsi@2:0.0.0\n"
" alias: primary_boot\n"
" type: 'VBOX HARDDISK'\n")
hw_profile = yaml.load(yaml_snippet)
hw_profile_model = HardwareProfile(**hw_profile)
assert hasattr(hw_profile_model, 'bootstrap_protocol')

BIN
tests/test_models.pyc Normal file
View File

Binary file not shown.

15
tox.ini Normal file
View File

@ -0,0 +1,15 @@
[tox]
envlist = py35
[testenv]
deps=
-rrequirements.txt
-rtestrequirements.txt
setenv=
PYTHONWARNING=all
commands=
py.test \
{posargs}
[flake8]
ignore=E302,H306