diff --git a/helm_drydock/config.py b/helm_drydock/config.py index bf364d1f..8bd363bc 100644 --- a/helm_drydock/config.py +++ b/helm_drydock/config.py @@ -22,7 +22,13 @@ class DrydockConfig(object): def __init__(self): - self.selected_server_driver = helm_drydock.drivers.server.maasdriver + self.server_driver_config = { + selected_driver = helm_drydock.drivers.server.maasdriver, + params = { + maas_api_key = "" + maas_api_url = "" + } + } self.selected_network_driver = helm_drydock.drivers.network.noopdriver self.control_config = {} self.ingester_config = { @@ -31,5 +37,5 @@ class DrydockConfig(object): self.introspection_config = {} self.orchestrator_config = {} self.statemgmt_config = { - backend_driver = 'helm_drydock.drivers.statemgmt.etcd', + backend_driver = helm_drydock.drivers.statemgmt.etcd, } diff --git a/helm_drydock/ingester/plugins/ingester_plugin.py b/helm_drydock/ingester/plugins/__init__.py similarity index 79% rename from helm_drydock/ingester/plugins/ingester_plugin.py rename to helm_drydock/ingester/plugins/__init__.py index 8d4dc955..7281f22e 100644 --- a/helm_drydock/ingester/plugins/ingester_plugin.py +++ b/helm_drydock/ingester/plugins/__init__.py @@ -19,12 +19,12 @@ import logging class IngesterPlugin(object): - def __init__(self): - self.log = logging.Logger('ingester') - return + def __init__(self): + self.log = logging.Logger('ingester') + return - def get_data(self): - return "ingester_skeleton" + def get_data(self): + return "ingester_skeleton" - def ingest_data(self, **kwargs): - return {} + def ingest_data(self, **kwargs): + return {} diff --git a/helm_drydock/ingester/plugins/aicyaml.py b/helm_drydock/ingester/plugins/aicyaml.py index c54c6eed..2da7bba2 100644 --- a/helm_drydock/ingester/plugins/aicyaml.py +++ b/helm_drydock/ingester/plugins/aicyaml.py @@ -14,42 +14,86 @@ # # AIC YAML Ingester - This data ingester will consume a AIC YAML design -# file -# +# file +# import yaml import logging -import helm_drydock.ingester.plugins.IngesterPlugin +import helm_drydock.model as model +from helm_drydock.ingester.plugins import IngesterPlugin class AicYamlIngester(IngesterPlugin): - def __init__(self): - super(AicYamlIngester, self).__init__() + kind_map = { + "Region": model.Site, + "NetworkLink": model.NetworkLink, + "HardwareProfile": model.HardwareProfile, + "Network": model.Network, + "HostProfile": model.HostProfile, + "BaremetalNode": model.BaremetalNode, + } - def get_name(self): - return "aic_yaml" + def __init__(self): + super(AicYamlIngester, self).__init__() - """ - AIC YAML ingester params + def get_name(self): + return "aic_yaml" - filename - Absolute path to the YAML file to ingest - """ - def ingest_data(self, **kwargs): - if 'filename' in params: - input_string = read_input_file(params['filename']) - parsed_data = parse_input_data(input_string) - processed_data = compute_effective_data(parsed_data) - else: + """ + AIC YAML ingester params - raise Exception('Missing parameter') - - return processed_data + filenames - Array of absolute path to the YAML files to ingest - def read_input_file(self, filename): - try: - file = open(filename,'rt') - except OSError as err: - self.log.error("Error opening input file %s for ingestion: %s" % (filename, err)) - return {} + returns an array of objects from helm_drydock.model + + """ + def ingest_data(self, **kwargs): + if 'filenames' in kwargs: + # TODO validate filenames is array + for f in kwargs.get('filenames'): + try: + file = open(f,'rt') + contents = file.read() + file.close() + except OSError as err: + self.log.error( + "Error opening input file %s for ingestion: %s" + % (filename, err)) + continue + + try: + parsed_data = yaml.load_all(contents) + except yaml.YAMLError as err: + self.log.error("Error parsing YAML in %s: %s" % (f, err)) + continue + + models = [] + for d in parsed_data: + kind = d.get('kind', '') + if kind != '': + if kind in AicYamlIngester.kind_map: + try: + model = AicYamlIngester.kind_map[kind](**d) + models.append(model) + except Exception as err: + self.log.error("Error building model %s: %s" + % (kind, str(err))) + else: + self.log.error( + "Error processing document, unknown kind %s" + % (kind)) + continue + else: + self.log.error( + "Error processing document in %s, no kind field" + % (f)) + continue + + return models + else: + raise ValueError('Missing parameter "filename"') + + return processed_data + diff --git a/setup.py b/setup.py index 870b0bb0..5107af4e 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,8 @@ setup(name='helm_drydock', license='Apache 2.0', packages=['helm_drydock', 'helm_drydock.model', - 'helm_drydock.ingester'], + 'helm_drydock.ingester', + 'helm_drydock.ingester.plugins'], install_requires=[ 'PyYAML', 'oauth', diff --git a/tests/aicyaml_samples/invalid.yaml b/tests/aicyaml_samples/invalid.yaml new file mode 100644 index 00000000..ef525833 --- /dev/null +++ b/tests/aicyaml_samples/invalid.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: +: Network +metadata: + name: public + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '102' + # How are addresses assigned? + allocation: static + # MTU size for the VLAN interface + mtu: 1500 + cidr: 172.16.3.0/24 + # Desribe IP address ranges + ranges: + - type: static + start: 172.16.3.15 + end: 172.16.3.254 + routes: + - subnet: 0.0.0.0/0 + gateway: 172.16.3.1 + metric: 9 + dns: + domain: sitename.example.com + servers: 8.8.8.8 \ No newline at end of file diff --git a/tests/aicyaml_samples/multidoc.yaml b/tests/aicyaml_samples/multidoc.yaml new file mode 100644 index 00000000..693afd1d --- /dev/null +++ b/tests/aicyaml_samples/multidoc.yaml @@ -0,0 +1,77 @@ +--- +apiVersion: '1.0' +kind: NetworkLink +metadata: + name: oob + region: sitename + date: 17-FEB-2017 + name: Sample network link + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: 100full + trunking: + mode: none + default_network: oob +--- +# pxe is a bit of 'magic' indicating the link config used when PXE booting +# a node. All other links indicate network configs applied when the node +# is deployed. +apiVersion: '1.0' +kind: NetworkLink +metadata: + name: pxe + region: sitename + date: 17-FEB-2017 + name: Sample network link + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + # none is a port-based VLAN identified by default_network + # tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr + trunking: + mode: none + # use name, will translate to VLAN ID + default_network: pxe +--- +apiVersion: '1.0' +kind: NetworkLink +metadata: + name: gp + region: sitename + date: 17-FEB-2017 + name: Sample network link + author: sh8121@att.com + description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on + # pxe is a bit of 'magic' indicating the link config used when PXE booting + # a node. All other links indicate network configs applied when the node + # is deployed. +spec: + # If this link is a bond of physical links, how is it configured + # 802.3ad + # active-backup + # balance-rr + # Can add support for others down the road + bonding: + mode: 802.3ad + # For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4 + hash: layer3+4 + # 802.3ad specific options + peer_rate: slow + mon_rate: default + up_delay: default + down_delay: default + mtu: 9000 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + trunking: + mode: tagged + default_network: mgmt \ No newline at end of file diff --git a/tests/aicyaml_samples/singledoc.yaml b/tests/aicyaml_samples/singledoc.yaml new file mode 100644 index 00000000..8ad16094 --- /dev/null +++ b/tests/aicyaml_samples/singledoc.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: '1.0' +kind: HardwareProfile +metadata: + name: HPGen8v3 + region: sitename + date: 17-FEB-2017 + name: Sample hardware definition + author: Scott Hussey +spec: + # Vendor of the server chassis + vendor: HP + # Generation of the chassis model + generation: '8' + # Version of the chassis model within its generation - not version of the hardware definition + hw_version: '3' + # The certified version of the chassis BIOS + bios_version: '2.2.3' + # Mode of the default boot of hardware - bios, uefi + boot_mode: bios + # Protocol of boot of the hardware - pxe, usb, hdd + bootstrap_protocol: pxe + # Which interface to use for network booting within the OOB manager, not OS device + pxe_interface: 0 + # Map hardware addresses to aliases/roles to allow a mix of hardware configs + # in a site to result in a consistent configuration + device_aliases: + pci: + - address: pci@0000:00:03.0 + alias: prim_nic01 + # type could identify expected hardware - used for hardware manifest validation + type: '82540EM Gigabit Ethernet Controller' + - address: pci@0000:00:04.0 + alias: prim_nic02 + type: '82540EM Gigabit Ethernet Controller' + scsi: + - address: scsi@2:0.0.0 + alias: primary_boot + type: 'VBOX HARDDISK' \ No newline at end of file diff --git a/tests/aicyaml_samples/unknown_kind.yaml b/tests/aicyaml_samples/unknown_kind.yaml new file mode 100644 index 00000000..bd39bfc0 --- /dev/null +++ b/tests/aicyaml_samples/unknown_kind.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: '1.0' +kind: FooBar +metadata: + name: default + region: sitename + date: 17-FEB-2017 + name: Sample network definition + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces + # No magic to this host_profile, it just provides a way to specify + # sitewide settings. If it is absent from a node's inheritance chain + # then these values will NOT be applied +spec: + # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such + # as IPMI over vender-specific when possible. + oob: + type: ipmi + # OOB networking should be preconfigured, but we can include a network + # definition for validation or enhancement (DNS registration) + network: oob + account: admin + credential: admin + # Specify storage layout of base OS. Ceph out of scope + storage: + # How storage should be carved up: lvm (logical volumes), flat + # (single partition) + layout: lvm + # Info specific to the boot and root disk/partitions + bootdisk: + # Device will specify an alias defined in hwdefinition.yaml + device: primary_boot + # For LVM, the size of the partition added to VG as a PV + # For flat, the size of the partition formatted as ext4 + root_size: 50g + # The /boot partition. If not specified, /boot will in root + boot_size: 2g + # Info for additional partitions. Need to balance between + # flexibility and complexity + partitions: + - name: logs + device: primary_boot + # Partition uuid if needed + part_uuid: 84db9664-f45e-11e6-823d-080027ef795a + size: 10g + # Optional, can carve up unformatted block devices + mountpoint: /var/log + fstype: ext4 + mount_options: defaults + # Filesystem UUID or label can be specified. UUID recommended + fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e + fs_label: logs + # Platform (Operating System) settings + platform: + image: ubuntu_16.04_hwe + kernel_params: default + # Additional metadata to apply to a node + metadata: + # Base URL of the introspection service - may go in curtin data + introspection_url: http://172.16.1.10:9090 \ No newline at end of file diff --git a/tests/test_ingester_aicyaml.py b/tests/test_ingester_aicyaml.py new file mode 100644 index 00000000..0827bc8b --- /dev/null +++ b/tests/test_ingester_aicyaml.py @@ -0,0 +1,54 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from helm_drydock.ingester.plugins.aicyaml import AicYamlIngester +import pytest +import shutil +import os + +class TestClass(object): + + def setup_method(self, method): + print("Running test {0}".format(method.__name__)) + + def test_ingest_singledoc(self, input_files): + input_file = input_files.join("singledoc.yaml") + + ingester = AicYamlIngester() + + models = ingester.ingest_data(filenames=[str(input_file)]) + + assert len(models) == 1 + + def test_ingest_multidoc(self, input_files): + input_file = input_files.join("multidoc.yaml") + + ingester = AicYamlIngester() + + models = ingester.ingest_data(filenames=[str(input_file)]) + + assert len(models) == 3 + + @pytest.fixture(scope='module') + def input_files(self, tmpdir_factory, request): + tmpdir = tmpdir_factory.mktemp('data') + samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples" + samples = os.listdir(samples_dir) + + for f in samples: + src_file = samples_dir + "/" + f + dst_file = str(tmpdir) + "/" + f + shutil.copyfile(src_file, dst_file) + + return tmpdir \ No newline at end of file