diff --git a/docs/source/index.rst b/docs/source/index.rst index ac1fef2e..9c36bb76 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -69,3 +69,4 @@ Topology Documentation :maxdepth: 1 topology + troubleshooting/index diff --git a/docs/source/troubleshooting/index.rst b/docs/source/troubleshooting/index.rst new file mode 100644 index 00000000..43cc6999 --- /dev/null +++ b/docs/source/troubleshooting/index.rst @@ -0,0 +1,40 @@ +.. + Copyright 2017 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +============================= +Dryodck Troubleshooting Guide +============================= + +This is a guide for troubleshooting issues that can arise when either +installing/deploying Drydock or using Drydock to deploy nodes. + +Deployment Troubleshooting +-------------------------- + +Under Construction + +Topology Validation +------------------- + +.. toctree:: + :maxdepth: 2 + + validations + +Node Deployment +--------------- + +Under Construction diff --git a/docs/source/troubleshooting/validations.rst b/docs/source/troubleshooting/validations.rst new file mode 100644 index 00000000..542882bc --- /dev/null +++ b/docs/source/troubleshooting/validations.rst @@ -0,0 +1,39 @@ +.. + Copyright 2018 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +============================= +Dryodck Topology Validation +============================= + +DD1XXX - Storage Validations +============================= + +To be continued + +DD2XXX - Network Validations +============================= + +To be continued + +DD3XXX - Platform Validations +============================= + +To be continued + +DD4XXX - Bootaction Validations +============================= + +To be continued diff --git a/drydock_provisioner/objects/bootaction.py b/drydock_provisioner/objects/bootaction.py index c9572a28..f472e069 100644 --- a/drydock_provisioner/objects/bootaction.py +++ b/drydock_provisioner/objects/bootaction.py @@ -50,6 +50,8 @@ class BootAction(base.DrydockPersistentObject, base.DrydockObject): def __init__(self, **kwargs): super().__init__(**kwargs) + if not self.target_nodes: + self.target_nodes = [] # NetworkLink keyed by name def get_id(self): @@ -123,9 +125,11 @@ class BootActionAsset(base.DrydockObject): mode = None ba_type = kwargs.get('type', None) + + package_list = None if ba_type == 'pkg_list': if isinstance(kwargs.get('data'), dict): - self._extract_package_list(kwargs.pop('data')) + package_list = self._extract_package_list(kwargs.pop('data')) # If the data section doesn't parse as a dictionary # then the package data needs to be sourced dynamically # Otherwise the Bootaction is invalid @@ -133,7 +137,7 @@ class BootActionAsset(base.DrydockObject): raise errors.InvalidPackageListFormat( "Requires a top-level mapping/object.") - super().__init__(permissions=mode, **kwargs) + super().__init__(package_list=package_list, permissions=mode, **kwargs) self.rendered_bytes = None def render(self, nodename, site_design, action_id, design_ref): @@ -180,7 +184,7 @@ class BootActionAsset(base.DrydockObject): parsed_data = yaml.safe_load(data_string) if isinstance(parsed_data, dict): - self._extract_package_list(parsed_data) + self.package_list = self._extract_package_list(parsed_data) else: raise errors.InvalidPackageListFormat( "Package data should have a top-level mapping/object.") @@ -193,13 +197,14 @@ class BootActionAsset(base.DrydockObject): :param pkg_dict: a dictionary of packages to install """ - self.package_list = dict() + package_list = dict() for k, v in pkg_dict.items(): - if isinstance(k, str) and isinstance(v, str): - self.package_list[k] = v + if (isinstance(k, str) and (not v or isinstance(v, str))): + package_list[k] = v else: raise errors.InvalidPackageListFormat( "Keys and values must be strings.") + return package_list def _get_template_context(self, nodename, site_design, action_id, design_ref): diff --git a/drydock_provisioner/orchestrator/validations/bootaction_validity.py b/drydock_provisioner/orchestrator/validations/bootaction_validity.py new file mode 100644 index 00000000..68da503f --- /dev/null +++ b/drydock_provisioner/orchestrator/validations/bootaction_validity.py @@ -0,0 +1,88 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +from drydock_provisioner import error as errors +from drydock_provisioner.orchestrator.validations.validators import Validators + + +class BootactionDefined(Validators): + """Issue warnings if no bootactions are defined for a node.""" + + def __init__(self): + super().__init__('Bootaction Definition', 'DD4001') + + def run_validation(self, site_design, orchestrator=None): + """Validate each node has at least one bootaction.""" + node_list = site_design.baremetal_nodes or [] + ba_list = site_design.bootactions or [] + + nodes_with_ba = [n for ba in ba_list for n in ba.target_nodes] + nodes_wo_ba = [n for n in node_list if n.name not in nodes_with_ba] + + for n in nodes_wo_ba: + msg = "Node %s is not in scope for any bootactions." % n.name + self.report_warn(msg, [ + n.doc_ref + ], "It is expected all nodes have at least one post-deploy action." + ) + return + + +class BootactionPackageListValid(Validators): + """Check that bootactions with pkg_list assets are valid.""" + + def __init__(self): + super().__init__('Bootaction pkg_list Validation', 'DD4002') + version_fields = '(\d+:)?([a-zA-Z0-9.+~-]+)(-[a-zA-Z0-9.+~]+)' + self.version_fields = re.compile(version_fields) + + def run_validation(self, site_design, orchestrator=None): + """Validate that each package list in bootaction assets is valid.""" + ba_list = site_design.bootactions or [] + + for ba in ba_list: + for a in ba.asset_list: + if a.type == 'pkg_list': + if not a.location and not a.package_list: + msg = "Bootaction has asset of type 'pkg_list' but no valid package data" + self.report_error(msg, [ + ba.doc_ref + ], "pkg_list bootaction assets must specify a list of packages." + ) + elif a.package_list: + for p, v in a.package_list.items(): + try: + self.validate_package_version(v) + except errors.InvalidPackageListFormat as ex: + msg = str(ex) + self.report_error(msg, [ + ba.doc_ref + ], "pkg_list version specifications must be in a valid format." + ) + return + + def validate_package_version(self, v): + """Validate that a version specification is valid. + + :param v: a string package specification. + """ + if not v: + return + + spec_match = self.version_fields.fullmatch(v) + + if not spec_match: + raise errors.InvalidPackageListFormat( + "Version specifier %s is not a valid format." % v) diff --git a/drydock_provisioner/orchestrator/validations/validator.py b/drydock_provisioner/orchestrator/validations/validator.py index ee5ae3b3..cfc70371 100644 --- a/drydock_provisioner/orchestrator/validations/validator.py +++ b/drydock_provisioner/orchestrator/validations/validator.py @@ -30,6 +30,8 @@ from drydock_provisioner.orchestrator.validations.unique_network_check import Un from drydock_provisioner.orchestrator.validations.hostname_validity import HostnameValidity from drydock_provisioner.orchestrator.validations.oob_valid_ipmi import IpmiValidity from drydock_provisioner.orchestrator.validations.oob_valid_libvirt import LibvirtValidity +from drydock_provisioner.orchestrator.validations.bootaction_validity import BootactionDefined +from drydock_provisioner.orchestrator.validations.bootaction_validity import BootactionPackageListValid class Validator(): @@ -91,5 +93,7 @@ rule_set = [ UniqueNetworkCheck(), HostnameValidity(), IpmiValidity(), - LibvirtValidity() + LibvirtValidity(), + BootactionDefined(), + BootactionPackageListValid(), ] diff --git a/tests/unit/test_validation_rule_bootactions.py b/tests/unit/test_validation_rule_bootactions.py new file mode 100644 index 00000000..9da5939c --- /dev/null +++ b/tests/unit/test_validation_rule_bootactions.py @@ -0,0 +1,92 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test Validation Rule Rational Boot Storage""" + +import logging + +from drydock_provisioner.orchestrator.orchestrator import Orchestrator +from drydock_provisioner import objects +from drydock_provisioner.objects import fields as hd_fields +from drydock_provisioner.orchestrator.validations.bootaction_validity import BootactionDefined + +LOG = logging.getLogger(__name__) + + +class TestBootactionsValidity(object): + def test_valid_bootaction(self, deckhand_ingester, drydock_state, setup, + input_files, mock_get_build_data): + input_file = input_files.join("validation.yaml") + design_ref = "file://%s" % str(input_file) + + orch = Orchestrator( + state_manager=drydock_state, ingester=deckhand_ingester) + + status, site_design = Orchestrator.get_effective_site(orch, design_ref) + + assert status.status == hd_fields.ValidationResult.Success + + LOG.debug("%s" % status.to_dict()) + + validator = BootactionDefined() + message_list = validator.execute(site_design, orchestrator=orch) + msg = message_list[0].to_dict() + + assert msg.get('error') is False + assert msg.get('level') == hd_fields.MessageLevels.INFO + assert len(message_list) == 1 + + def test_absent_bootaction(self, deckhand_ingester, drydock_state, setup, + input_files, mock_get_build_data): + input_file = input_files.join("absent_bootaction.yaml") + design_ref = "file://%s" % str(input_file) + + orch = Orchestrator( + state_manager=drydock_state, ingester=deckhand_ingester) + + status, site_design = Orchestrator.get_effective_site(orch, design_ref) + + ba_msg = [ + msg for msg in status.message_list + if ("DD4001" in msg.name + and msg.level == hd_fields.MessageLevels.WARN) + ] + + assert len(ba_msg) > 0 + + def test_invalid_bootaction_pkg_list(self, deckhand_ingester, + drydock_state, setup, input_files, + mock_get_build_data): + input_file = input_files.join("invalid_bootaction_pkg.yaml") + design_ref = "file://%s" % str(input_file) + + orch = Orchestrator( + state_manager=drydock_state, ingester=deckhand_ingester) + + status, site_design = Orchestrator.get_effective_site(orch, design_ref) + + ba_doc = objects.DocumentReference( + doc_type=hd_fields.DocumentType.Deckhand, + doc_name="invalid_pkg_list", + doc_schema="drydock/BootAction/v1") + + assert status.status == hd_fields.ValidationResult.Failure + + ba_msg = [msg for msg in status.message_list if ba_doc in msg.docs] + + assert len(ba_msg) > 0 + + for msg in ba_msg: + LOG.debug(msg) + assert ":) is not a valid format" in msg.message + assert msg.error diff --git a/tests/yaml_samples/absent_bootaction.yaml b/tests/yaml_samples/absent_bootaction.yaml new file mode 100644 index 00000000..d0f84304 --- /dev/null +++ b/tests/yaml_samples/absent_bootaction.yaml @@ -0,0 +1,128 @@ +#Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +schema: 'drydock/HostProfile/v1' +metadata: + schema: 'metadata/Document/v1' + name: defaults + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + hardware_profile: HPGen9v3 + oob: + type: ipmi + network: oob + account: admin + credential: admin + storage: + physical_devices: + sda: + labels: + role: rootdisk + partitions: + - name: root + size: 20g + bootable: true + filesystem: + mountpoint: '/' + fstype: 'ext4' + mount_options: 'defaults' + - name: boot + size: 1g + bootable: false + filesystem: + mountpoint: '/boot' + fstype: 'ext4' + mount_options: 'defaults' + sdb: + volume_group: 'log_vg' + volume_groups: + log_vg: + logical_volumes: + - name: 'log_lv' + size: '500m' + filesystem: + mountpoint: '/var/log' + fstype: 'xfs' + mount_options: 'defaults' + platform: + image: 'xenial' + kernel: 'ga-16.04' + kernel_params: + quiet: true + console: ttyS2 + metadata: + owner_data: + foo: bar +--- +schema: 'drydock/BaremetalNode/v1' +metadata: + schema: 'metadata/Document/v1' + name: controller01 + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + host_profile: defaults + addressing: + - network: pxe + address: dhcp + - network: mgmt + address: 172.16.1.20 + - network: public + address: 172.16.3.20 + - network: oob + address: 172.16.100.20 + metadata: + rack: rack1 +--- +schema: 'drydock/HardwareProfile/v1' +metadata: + schema: 'metadata/Document/v1' + name: HPGen9v3 + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + vendor: HP + generation: '8' + hw_version: '3' + bios_version: '2.2.3' + boot_mode: bios + bootstrap_protocol: pxe + pxe_interface: 0 + device_aliases: + prim_nic01: + address: '0000:00:03.0' + dev_type: '82540EM Gigabit Ethernet Controller' + bus_type: 'pci' + prim_nic02: + address: '0000:00:04.0' + dev_type: '82540EM Gigabit Ethernet Controller' + bus_type: 'pci' + primary_boot: + address: '2:0.0.0' + dev_type: 'VBOX HARDDISK' + bus_type: 'scsi' + cpu_sets: + sriov: '2,4' + hugepages: + sriov: + size: '1G' + count: 300 + dpdk: + size: '2M' + count: 530000 +... diff --git a/tests/yaml_samples/deckhand_fullsite_no_nodes.yaml b/tests/yaml_samples/deckhand_fullsite_no_nodes.yaml index 35b59bfa..c82d5fba 100644 --- a/tests/yaml_samples/deckhand_fullsite_no_nodes.yaml +++ b/tests/yaml_samples/deckhand_fullsite_no_nodes.yaml @@ -346,80 +346,4 @@ data: dpdk: size: '2M' count: 530000 ---- -schema: 'drydock/BootAction/v1' -metadata: - schema: 'metadata/Document/v1' - name: hw_filtered - storagePolicy: 'cleartext' - labels: - application: 'drydock' -data: - signaling: false - node_filter: - filter_set_type: 'union' - filter_set: - - filter_type: 'union' - node_names: - - 'compute01' - assets: - - path: /var/tmp/hello.sh - type: file - permissions: '555' - data: |- - IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19 - Jwo= - data_pipeline: - - base64_decode - - utf8_decode - - template - - path: /lib/systemd/system/hello.service - type: unit - permissions: '600' - data: |- - W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4 - ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu - dGFyZ2V0Cg== - data_pipeline: - - base64_decode - - utf8_decode -... ---- -schema: 'drydock/BootAction/v1' -metadata: - schema: 'metadata/Document/v1' - name: helloworld - storagePolicy: 'cleartext' - labels: - application: 'drydock' -data: - assets: - - path: /var/tmp/hello.sh - type: file - permissions: '555' - data: |- - IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19 - Jwo= - data_pipeline: - - base64_decode - - utf8_decode - - template - - path: /lib/systemd/system/hello.service - type: unit - permissions: '600' - data: |- - W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4 - ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu - dGFyZ2V0Cg== - data_pipeline: - - base64_decode - - utf8_decode - - path: /var/tmp/designref.sh - type: file - permissions: '500' - data: e3sgYWN0aW9uLmRlc2lnbl9yZWYgfX0K - data_pipeline: - - base64_decode - - utf8_decode - - template ... diff --git a/tests/yaml_samples/invalid_bootaction_pkg.yaml b/tests/yaml_samples/invalid_bootaction_pkg.yaml new file mode 100644 index 00000000..fb7d1cc1 --- /dev/null +++ b/tests/yaml_samples/invalid_bootaction_pkg.yaml @@ -0,0 +1,14 @@ +--- +schema: 'drydock/BootAction/v1' +metadata: + schema: 'metadata/Document/v1' + name: invalid_pkg_list + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + assets: + - type: 'pkg_list' + data: + foo: ':)' +...