diff --git a/releasenotes/notes/build-baremetal-data-model-in-watcher-3023453a47b61dab.yaml b/releasenotes/notes/build-baremetal-data-model-in-watcher-3023453a47b61dab.yaml new file mode 100644 index 000000000..93d889dfc --- /dev/null +++ b/releasenotes/notes/build-baremetal-data-model-in-watcher-3023453a47b61dab.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds baremetal data model in Watcher diff --git a/setup.cfg b/setup.cfg index 588557179..65d61ad33 100644 --- a/setup.cfg +++ b/setup.cfg @@ -97,6 +97,7 @@ watcher_planners = watcher_cluster_data_model_collectors = compute = watcher.decision_engine.model.collector.nova:NovaClusterDataModelCollector storage = watcher.decision_engine.model.collector.cinder:CinderClusterDataModelCollector + baremetal = watcher.decision_engine.model.collector.ironic:BaremetalClusterDataModelCollector [pbr] diff --git a/watcher/common/exception.py b/watcher/common/exception.py index afae2f307..ab7de4f87 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -473,6 +473,14 @@ class VolumeNotFound(StorageResourceNotFound): msg_fmt = _("The volume '%(name)s' could not be found") +class BaremetalResourceNotFound(WatcherException): + msg_fmt = _("The baremetal resource '%(name)s' could not be found") + + +class IronicNodeNotFound(BaremetalResourceNotFound): + msg_fmt = _("The ironic node %(uuid)s could not be found") + + class LoadingError(WatcherException): msg_fmt = _("Error loading plugin '%(name)s'") diff --git a/watcher/common/ironic_helper.py b/watcher/common/ironic_helper.py new file mode 100644 index 000000000..e8b0b56a4 --- /dev/null +++ b/watcher/common/ironic_helper.py @@ -0,0 +1,49 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2017 ZTE Corporation +# +# Authors:Yumeng Bao + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from oslo_log import log + +from watcher.common import clients +from watcher.common import exception +from watcher.common import utils + +LOG = log.getLogger(__name__) + + +class IronicHelper(object): + + def __init__(self, osc=None): + """:param osc: an OpenStackClients instance""" + self.osc = osc if osc else clients.OpenStackClients() + self.ironic = self.osc.ironic() + + def get_ironic_node_list(self): + return self.ironic.node.list() + + def get_ironic_node_by_uuid(self, node_uuid): + """Get ironic node by node UUID""" + try: + node = self.ironic.node.get(utils.Struct(uuid=node_uuid)) + if not node: + raise exception.IronicNodeNotFound(uuid=node_uuid) + except Exception as exc: + LOG.exception(exc) + raise exception.IronicNodeNotFound(uuid=node_uuid) + # We need to pass an object with an 'uuid' attribute to make it work + return node diff --git a/watcher/decision_engine/model/collector/ironic.py b/watcher/decision_engine/model/collector/ironic.py new file mode 100644 index 000000000..e8af73a4c --- /dev/null +++ b/watcher/decision_engine/model/collector/ironic.py @@ -0,0 +1,97 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2017 ZTE Corporation +# +# Authors:Yumeng Bao +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_log import log + +from watcher.common import ironic_helper +from watcher.decision_engine.model.collector import base +from watcher.decision_engine.model import element +from watcher.decision_engine.model import model_root + +LOG = log.getLogger(__name__) + + +class BaremetalClusterDataModelCollector(base.BaseClusterDataModelCollector): + """Baremetal cluster data model collector + + The Baremetal cluster data model collector creates an in-memory + representation of the resources exposed by the baremetal service. + """ + + def __init__(self, config, osc=None): + super(BaremetalClusterDataModelCollector, self).__init__(config, osc) + + @property + def notification_endpoints(self): + """Associated notification endpoints + + :return: Associated notification endpoints + :rtype: List of :py:class:`~.EventsNotificationEndpoint` instances + """ + return None + + def get_audit_scope_handler(self, audit_scope): + return None + + def execute(self): + """Build the baremetal cluster data model""" + LOG.debug("Building latest Baremetal cluster data model") + + builder = ModelBuilder(self.osc) + return builder.execute() + + +class ModelBuilder(object): + """Build the graph-based model + + This model builder adds the following data" + + - Baremetal-related knowledge (Ironic) + """ + def __init__(self, osc): + self.osc = osc + self.model = model_root.BaremetalModelRoot() + self.ironic_helper = ironic_helper.IronicHelper(osc=self.osc) + + def add_ironic_node(self, node): + # Build and add base node. + ironic_node = self.build_ironic_node(node) + self.model.add_node(ironic_node) + + def build_ironic_node(self, node): + """Build a Baremetal node from a Ironic node + + :param node: A ironic node + :type node: :py:class:`~ironicclient.v1.node.Node` + """ + # build up the ironic node. + node_attributes = { + "uuid": node.uuid, + "power_state": node.power_state, + "maintenance": node.maintenance, + "maintenance_reason": node.maintenance_reason, + "extra": {"compute_node_id": node.extra.compute_node_id} + } + + ironic_node = element.IronicNode(**node_attributes) + return ironic_node + + def execute(self): + + for node in self.ironic_helper.get_ironic_node_list(): + self.add_ironic_node(node) + return self.model diff --git a/watcher/decision_engine/model/element/__init__.py b/watcher/decision_engine/model/element/__init__.py index dce25287f..d80596da6 100644 --- a/watcher/decision_engine/model/element/__init__.py +++ b/watcher/decision_engine/model/element/__init__.py @@ -23,6 +23,7 @@ from watcher.decision_engine.model.element import volume ServiceState = node.ServiceState ComputeNode = node.ComputeNode StorageNode = node.StorageNode +IronicNode = node.IronicNode Pool = node.Pool InstanceState = instance.InstanceState @@ -37,4 +38,5 @@ __all__ = ['ServiceState', 'StorageNode', 'Pool', 'VolumeState', - 'Volume'] + 'Volume', + 'IronicNode'] diff --git a/watcher/decision_engine/model/element/baremetal_resource.py b/watcher/decision_engine/model/element/baremetal_resource.py new file mode 100644 index 000000000..d53640fe5 --- /dev/null +++ b/watcher/decision_engine/model/element/baremetal_resource.py @@ -0,0 +1,33 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2017 ZTE Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc + +import six + +from watcher.decision_engine.model.element import base +from watcher.objects import fields as wfields + + +@six.add_metaclass(abc.ABCMeta) +class BaremetalResource(base.Element): + + VERSION = '1.0' + + fields = { + "uuid": wfields.StringField(), + "human_id": wfields.StringField(default=""), + } diff --git a/watcher/decision_engine/model/element/node.py b/watcher/decision_engine/model/element/node.py index b686dae92..4eab1b6ac 100644 --- a/watcher/decision_engine/model/element/node.py +++ b/watcher/decision_engine/model/element/node.py @@ -16,6 +16,7 @@ import enum +from watcher.decision_engine.model.element import baremetal_resource from watcher.decision_engine.model.element import compute_resource from watcher.decision_engine.model.element import storage_resource from watcher.objects import base @@ -78,3 +79,17 @@ class Pool(storage_resource.StorageResource): def accept(self, visitor): raise NotImplementedError() + + +@base.WatcherObjectRegistry.register_if(False) +class IronicNode(baremetal_resource.BaremetalResource): + + fields = { + "power_state": wfields.StringField(), + "maintenance": wfields.BooleanField(), + "maintenance_reason": wfields.StringField(), + "extra": wfields.DictField() + } + + def accept(self, visitor): + raise NotImplementedError() diff --git a/watcher/decision_engine/model/model_root.py b/watcher/decision_engine/model/model_root.py index 17e1db656..1dd710680 100644 --- a/watcher/decision_engine/model/model_root.py +++ b/watcher/decision_engine/model/model_root.py @@ -545,3 +545,85 @@ class StorageModelRoot(nx.DiGraph, base.Model): def is_isomorphic(cls, G1, G2): return nx.algorithms.isomorphism.isomorph.is_isomorphic( G1, G2) + + +class BaremetalModelRoot(nx.DiGraph, base.Model): + + """Cluster graph for an Openstack cluster: Baremetal Cluster.""" + + def __init__(self, stale=False): + super(BaremetalModelRoot, self).__init__() + self.stale = stale + + def __nonzero__(self): + return not self.stale + + __bool__ = __nonzero__ + + @staticmethod + def assert_node(obj): + if not isinstance(obj, element.IronicNode): + raise exception.IllegalArgumentException( + message=_("'obj' argument type is not valid: %s") % type(obj)) + + @lockutils.synchronized("baremetal_model") + def add_node(self, node): + self.assert_node(node) + super(BaremetalModelRoot, self).add_node(node.uuid, node) + + @lockutils.synchronized("baremetal_model") + def remove_node(self, node): + self.assert_node(node) + try: + super(BaremetalModelRoot, self).remove_node(node.uuid) + except nx.NetworkXError as exc: + LOG.exception(exc) + raise exception.IronicNodeNotFound(name=node.uuid) + + @lockutils.synchronized("baremetal_model") + def get_all_ironic_nodes(self): + return {uuid: cn for uuid, cn in self.nodes(data=True) + if isinstance(cn, element.IronicNode)} + + @lockutils.synchronized("baremetal_model") + def get_node_by_uuid(self, uuid): + try: + return self._get_by_uuid(uuid) + except exception.BaremetalResourceNotFound: + raise exception.IronicNodeNotFound(name=uuid) + + def _get_by_uuid(self, uuid): + try: + return self.node[uuid] + except Exception as exc: + LOG.exception(exc) + raise exception.BaremetalResourceNotFound(name=uuid) + + def to_string(self): + return self.to_xml() + + def to_xml(self): + root = etree.Element("ModelRoot") + # Build Ironic node tree + for cn in sorted(self.get_all_ironic_nodes().values(), + key=lambda cn: cn.uuid): + ironic_node_el = cn.as_xml_element() + root.append(ironic_node_el) + + return etree.tostring(root, pretty_print=True).decode('utf-8') + + @classmethod + def from_xml(cls, data): + model = cls() + + root = etree.fromstring(data) + for cn in root.findall('.//IronicNode'): + node = element.IronicNode(**cn.attrib) + model.add_node(node) + + return model + + @classmethod + def is_isomorphic(cls, G1, G2): + return nx.algorithms.isomorphism.isomorph.is_isomorphic( + G1, G2) diff --git a/watcher/decision_engine/scope/baremetal.py b/watcher/decision_engine/scope/baremetal.py new file mode 100644 index 000000000..1a090ca81 --- /dev/null +++ b/watcher/decision_engine/scope/baremetal.py @@ -0,0 +1,46 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2018 ZTE Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_log import log + +from watcher.decision_engine.scope import base + + +LOG = log.getLogger(__name__) + + +class BaremetalScope(base.BaseScope): + """Baremetal Audit Scope Handler""" + + def __init__(self, scope, config, osc=None): + super(BaremetalScope, self).__init__(scope, config) + self._osc = osc + + def get_scoped_model(self, cluster_model): + """Leave only nodes and instances proposed in the audit scope""" + if not cluster_model: + return None + + for scope in self.scope: + baremetal_scope = scope.get('baremetal') + + if not baremetal_scope: + return cluster_model + + # TODO(yumeng-bao): currently self.scope is always [] + # Audit scoper for baremetal data model will be implemented: + # https://blueprints.launchpad.net/watcher/+spec/audit-scoper-for-baremetal-data-model + return cluster_model diff --git a/watcher/decision_engine/strategy/strategies/base.py b/watcher/decision_engine/strategy/strategies/base.py index 50003b571..13750aa08 100644 --- a/watcher/decision_engine/strategy/strategies/base.py +++ b/watcher/decision_engine/strategy/strategies/base.py @@ -85,6 +85,7 @@ class BaseStrategy(loadable.Loadable): self._collector_manager = None self._compute_model = None self._storage_model = None + self._baremetal_model = None self._input_parameters = utils.Struct() self._audit_scope = None self._datasource_backend = None @@ -219,6 +220,29 @@ class BaseStrategy(loadable.Loadable): return self._storage_model + @property + def baremetal_model(self): + """Cluster data model + + :returns: Cluster data model the strategy is executed on + :rtype model: :py:class:`~.ModelRoot` instance + """ + if self._baremetal_model is None: + collector = self.collector_manager.get_cluster_model_collector( + 'baremetal', osc=self.osc) + audit_scope_handler = collector.get_audit_scope_handler( + audit_scope=self.audit_scope) + self._baremetal_model = audit_scope_handler.get_scoped_model( + collector.get_latest_cluster_data_model()) + + if not self._baremetal_model: + raise exception.ClusterStateNotDefined() + + if self._baremetal_model.stale: + raise exception.ClusterStateStale() + + return self._baremetal_model + @classmethod def get_schema(cls): """Defines a Schema that the input parameters shall comply to diff --git a/watcher/tests/common/test_ironic_helper.py b/watcher/tests/common/test_ironic_helper.py new file mode 100644 index 000000000..d51cc285e --- /dev/null +++ b/watcher/tests/common/test_ironic_helper.py @@ -0,0 +1,63 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2017 ZTE Corporation +# +# Authors:Yumeng Bao + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import mock + +from watcher.common import clients +from watcher.common import exception +from watcher.common import ironic_helper +from watcher.common import utils as w_utils +from watcher.tests import base + + +class TestIronicHelper(base.TestCase): + + def setUp(self): + super(TestIronicHelper, self).setUp() + + osc = clients.OpenStackClients() + p_ironic = mock.patch.object(osc, 'ironic') + p_ironic.start() + self.addCleanup(p_ironic.stop) + self.ironic_util = ironic_helper.IronicHelper(osc=osc) + + @staticmethod + def fake_ironic_node(): + node = mock.MagicMock() + node.uuid = w_utils.generate_uuid() + return node + + def test_get_ironic_node_list(self): + node1 = self.fake_ironic_node() + self.ironic_util.ironic.node.list.return_value = [node1] + rt_nodes = self.ironic_util.get_ironic_node_list() + self.assertEqual(rt_nodes, [node1]) + + def test_get_ironic_node_by_uuid_success(self): + node1 = self.fake_ironic_node() + self.ironic_util.ironic.node.get.return_value = node1 + node = self.ironic_util.get_ironic_node_by_uuid(node1.uuid) + self.assertEqual(node, node1) + + def test_get_ironic_node_by_uuid_failure(self): + self.ironic_util.ironic.node.get.return_value = None + self.assertRaisesRegex( + exception.IronicNodeNotFound, + "The ironic node node1 could not be found", + self.ironic_util.get_ironic_node_by_uuid, 'node1') diff --git a/watcher/tests/decision_engine/model/data/ironic_scenario_1.xml b/watcher/tests/decision_engine/model/data/ironic_scenario_1.xml new file mode 100644 index 000000000..f58755dc7 --- /dev/null +++ b/watcher/tests/decision_engine/model/data/ironic_scenario_1.xml @@ -0,0 +1,12 @@ + + + + 1 + + + + + 2 + + + diff --git a/watcher/tests/decision_engine/model/faker_cluster_state.py b/watcher/tests/decision_engine/model/faker_cluster_state.py index 2d24e69b0..6ae430af9 100644 --- a/watcher/tests/decision_engine/model/faker_cluster_state.py +++ b/watcher/tests/decision_engine/model/faker_cluster_state.py @@ -20,6 +20,7 @@ import os import mock +from watcher.common import utils from watcher.decision_engine.model.collector import base from watcher.decision_engine.model import element from watcher.decision_engine.model import model_root as modelroot @@ -261,3 +262,55 @@ class FakerStorageModelCollector(base.BaseClusterDataModelCollector): def generate_scenario_1(self): return self.load_model('storage_scenario_1.xml') + + +class FakerBaremetalModelCollector(base.BaseClusterDataModelCollector): + + def __init__(self, config=None, osc=None): + if config is None: + config = mock.Mock(period=777) + super(FakerBaremetalModelCollector, self).__init__(config) + + @property + def notification_endpoints(self): + return [] + + def get_audit_scope_handler(self, audit_scope): + return None + + def load_data(self, filename): + cwd = os.path.abspath(os.path.dirname(__file__)) + data_folder = os.path.join(cwd, "data") + + with open(os.path.join(data_folder, filename), 'rb') as xml_file: + xml_data = xml_file.read() + + return xml_data + + def load_model(self, filename): + return modelroot.BaremetalModelRoot.from_xml(self.load_data(filename)) + + def execute(self): + return self._cluster_data_model or self.build_scenario_1() + + def build_scenario_1(self): + model = modelroot.BaremetalModelRoot() + # number of nodes + node_count = 2 + + for i in range(0, node_count): + uuid = utils.generate_uuid() + node_attributes = { + "uuid": uuid, + "power_state": "power on", + "maintenance": "false", + "maintenance_reason": "null", + "extra": {"compute_node_id": i} + } + node = element.IronicNode(**node_attributes) + model.add_node(node) + + return model + + def generate_scenario_1(self): + return self.load_model('ironic_scenario_1.xml') diff --git a/watcher/tests/decision_engine/model/test_element.py b/watcher/tests/decision_engine/model/test_element.py index ba4a71599..aad2386bf 100644 --- a/watcher/tests/decision_engine/model/test_element.py +++ b/watcher/tests/decision_engine/model/test_element.py @@ -152,3 +152,30 @@ class TestStorageElement(base.TestCase): def test_as_xml_element(self): el = self.cls(**self.data) el.as_xml_element() + + +class TestIronicElement(base.TestCase): + + scenarios = [ + ("IronicNode_with_all_fields", dict( + cls=element.IronicNode, + data={ + "uuid": 'FAKE_UUID', + "power_state": 'up', + "maintenance": "false", + "maintenance_reason": "null", + "extra": {"compute_node_id": 1} + })), + ("IronicNode_with_some_fields", dict( + cls=element.IronicNode, + data={ + "uuid": 'FAKE_UUID', + "power_state": 'up', + "maintenance": "false", + "extra": {"compute_node_id": 1} + })), + ] + + def test_as_xml_element(self): + el = self.cls(**self.data) + el.as_xml_element() diff --git a/watcher/tests/decision_engine/model/test_model.py b/watcher/tests/decision_engine/model/test_model.py index c4cacb945..611aff6af 100644 --- a/watcher/tests/decision_engine/model/test_model.py +++ b/watcher/tests/decision_engine/model/test_model.py @@ -367,3 +367,73 @@ class TestStorageModel(base.TestCase): self.assertEqual(volume, model.get_volume_by_uuid(uuid_)) model.map_volume(volume, pool) self.assertEqual([volume], model.get_pool_volumes(pool)) + + +class TestBaremetalModel(base.TestCase): + + def load_data(self, filename): + cwd = os.path.abspath(os.path.dirname(__file__)) + data_folder = os.path.join(cwd, "data") + + with open(os.path.join(data_folder, filename), 'rb') as xml_file: + xml_data = xml_file.read() + + return xml_data + + def load_model(self, filename): + return model_root.StorageModelRoot.from_xml(self.load_data(filename)) + + def test_model_structure(self): + fake_cluster = faker_cluster_state.FakerBaremetalModelCollector() + model1 = fake_cluster.build_scenario_1() + self.assertEqual(2, len(model1.get_all_ironic_nodes())) + + expected_struct_str = self.load_data('ironic_scenario_1.xml') + model2 = model_root.BaremetalModelRoot.from_xml(expected_struct_str) + self.assertTrue( + model_root.BaremetalModelRoot.is_isomorphic(model2, model1)) + + def test_build_model_from_xml(self): + fake_cluster = faker_cluster_state.FakerBaremetalModelCollector() + + expected_model = fake_cluster.generate_scenario_1() + struct_str = self.load_data('ironic_scenario_1.xml') + + model = model_root.BaremetalModelRoot.from_xml(struct_str) + self.assertEqual(expected_model.to_string(), model.to_string()) + + def test_assert_node_raise(self): + model = model_root.BaremetalModelRoot() + node_uuid = uuidutils.generate_uuid() + node = element.IronicNode(uuid=node_uuid) + model.add_node(node) + self.assertRaises(exception.IllegalArgumentException, + model.assert_node, "obj") + + def test_add_node(self): + model = model_root.BaremetalModelRoot() + node_uuid = uuidutils.generate_uuid() + node = element.IronicNode(uuid=node_uuid) + model.add_node(node) + self.assertEqual(node, model.get_node_by_uuid(node_uuid)) + + def test_remove_node(self): + model = model_root.BaremetalModelRoot() + node_uuid = uuidutils.generate_uuid() + node = element.IronicNode(uuid=node_uuid) + model.add_node(node) + self.assertEqual(node, model.get_node_by_uuid(node_uuid)) + model.remove_node(node) + self.assertRaises(exception.IronicNodeNotFound, + model.get_node_by_uuid, node_uuid) + + def test_get_all_ironic_nodes(self): + model = model_root.BaremetalModelRoot() + for i in range(10): + node_uuid = uuidutils.generate_uuid() + node = element.IronicNode(uuid=node_uuid) + model.add_node(node) + all_nodes = model.get_all_ironic_nodes() + for node_uuid in all_nodes: + node = model.get_node_by_uuid(node_uuid) + model.assert_node(node)