Add baremetal data model

Change-Id: I57b7bb53b3bc84ad383ae485069274f5c5362c50
Implements: blueprint build-baremetal-data-model-in-watcher

releasenotes/notes/build-baremetal-data-model-in-watcher-3023453a47b61dab.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Adds baremetal data model in Watcher

setup.cfg

@@ -97,6 +97,7 @@ watcher_planners =
 watcher_cluster_data_model_collectors =
     compute = watcher.decision_engine.model.collector.nova:NovaClusterDataModelCollector
     storage = watcher.decision_engine.model.collector.cinder:CinderClusterDataModelCollector
+    baremetal = watcher.decision_engine.model.collector.ironic:BaremetalClusterDataModelCollector
 
 
 [pbr]

watcher/common/exception.py

@@ -473,6 +473,14 @@ class VolumeNotFound(StorageResourceNotFound):
     msg_fmt = _("The volume '%(name)s' could not be found")
 
 
+class BaremetalResourceNotFound(WatcherException):
+    msg_fmt = _("The baremetal resource '%(name)s' could not be found")
+
+
+class IronicNodeNotFound(BaremetalResourceNotFound):
+    msg_fmt = _("The ironic node %(uuid)s could not be found")
+
+
 class LoadingError(WatcherException):
     msg_fmt = _("Error loading plugin '%(name)s'")
 

watcher/common/ironic_helper.py

@@ -0,0 +1,49 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2017 ZTE Corporation
+#
+# Authors:Yumeng Bao <bao.yumeng@zte.com.cn>
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from oslo_log import log
+
+from watcher.common import clients
+from watcher.common import exception
+from watcher.common import utils
+
+LOG = log.getLogger(__name__)
+
+
+class IronicHelper(object):
+
+    def __init__(self, osc=None):
+        """:param osc: an OpenStackClients instance"""
+        self.osc = osc if osc else clients.OpenStackClients()
+        self.ironic = self.osc.ironic()
+
+    def get_ironic_node_list(self):
+        return self.ironic.node.list()
+
+    def get_ironic_node_by_uuid(self, node_uuid):
+        """Get ironic node by node UUID"""
+        try:
+            node = self.ironic.node.get(utils.Struct(uuid=node_uuid))
+            if not node:
+                raise exception.IronicNodeNotFound(uuid=node_uuid)
+        except Exception as exc:
+            LOG.exception(exc)
+            raise exception.IronicNodeNotFound(uuid=node_uuid)
+        # We need to pass an object with an 'uuid' attribute to make it work
+        return node

watcher/decision_engine/model/collector/ironic.py

@@ -0,0 +1,97 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2017 ZTE Corporation
+#
+# Authors:Yumeng Bao <bao.yumeng@zte.com.cn>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_log import log
+
+from watcher.common import ironic_helper
+from watcher.decision_engine.model.collector import base
+from watcher.decision_engine.model import element
+from watcher.decision_engine.model import model_root
+
+LOG = log.getLogger(__name__)
+
+
+class BaremetalClusterDataModelCollector(base.BaseClusterDataModelCollector):
+    """Baremetal cluster data model collector
+
+    The Baremetal cluster data model collector creates an in-memory
+    representation of the resources exposed by the baremetal service.
+    """
+
+    def __init__(self, config, osc=None):
+        super(BaremetalClusterDataModelCollector, self).__init__(config, osc)
+
+    @property
+    def notification_endpoints(self):
+        """Associated notification endpoints
+
+        :return: Associated notification endpoints
+        :rtype: List of :py:class:`~.EventsNotificationEndpoint` instances
+        """
+        return None
+
+    def get_audit_scope_handler(self, audit_scope):
+        return None
+
+    def execute(self):
+        """Build the baremetal cluster data model"""
+        LOG.debug("Building latest Baremetal cluster data model")
+
+        builder = ModelBuilder(self.osc)
+        return builder.execute()
+
+
+class ModelBuilder(object):
+    """Build the graph-based model
+
+    This model builder adds the following data"
+
+    - Baremetal-related knowledge (Ironic)
+    """
+    def __init__(self, osc):
+        self.osc = osc
+        self.model = model_root.BaremetalModelRoot()
+        self.ironic_helper = ironic_helper.IronicHelper(osc=self.osc)
+
+    def add_ironic_node(self, node):
+        # Build and add base node.
+        ironic_node = self.build_ironic_node(node)
+        self.model.add_node(ironic_node)
+
+    def build_ironic_node(self, node):
+        """Build a Baremetal node from a Ironic node
+
+        :param node: A ironic node
+        :type node: :py:class:`~ironicclient.v1.node.Node`
+        """
+        # build up the ironic node.
+        node_attributes = {
+            "uuid": node.uuid,
+            "power_state": node.power_state,
+            "maintenance": node.maintenance,
+            "maintenance_reason": node.maintenance_reason,
+            "extra": {"compute_node_id": node.extra.compute_node_id}
+            }
+
+        ironic_node = element.IronicNode(**node_attributes)
+        return ironic_node
+
+    def execute(self):
+
+        for node in self.ironic_helper.get_ironic_node_list():
+            self.add_ironic_node(node)
+        return self.model

watcher/decision_engine/model/element/__init__.py

@@ -23,6 +23,7 @@ from watcher.decision_engine.model.element import volume
 ServiceState = node.ServiceState
 ComputeNode = node.ComputeNode
 StorageNode = node.StorageNode
+IronicNode = node.IronicNode
 Pool = node.Pool
 
 InstanceState = instance.InstanceState
@@ -37,4 +38,5 @@ __all__ = ['ServiceState',
            'StorageNode',
            'Pool',
            'VolumeState',
-           'Volume']
+           'Volume',
+           'IronicNode']

watcher/decision_engine/model/element/baremetal_resource.py

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2017 ZTE Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+
+import six
+
+from watcher.decision_engine.model.element import base
+from watcher.objects import fields as wfields
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaremetalResource(base.Element):
+
+    VERSION = '1.0'
+
+    fields = {
+        "uuid": wfields.StringField(),
+        "human_id": wfields.StringField(default=""),
+    }

watcher/decision_engine/model/element/node.py

@@ -16,6 +16,7 @@
 
 import enum
 
+from watcher.decision_engine.model.element import baremetal_resource
 from watcher.decision_engine.model.element import compute_resource
 from watcher.decision_engine.model.element import storage_resource
 from watcher.objects import base
@@ -78,3 +79,17 @@ class Pool(storage_resource.StorageResource):
 
     def accept(self, visitor):
         raise NotImplementedError()
+
+
+@base.WatcherObjectRegistry.register_if(False)
+class IronicNode(baremetal_resource.BaremetalResource):
+
+    fields = {
+        "power_state": wfields.StringField(),
+        "maintenance": wfields.BooleanField(),
+        "maintenance_reason": wfields.StringField(),
+        "extra": wfields.DictField()
+    }
+
+    def accept(self, visitor):
+        raise NotImplementedError()

watcher/decision_engine/model/model_root.py

@@ -545,3 +545,85 @@ class StorageModelRoot(nx.DiGraph, base.Model):
     def is_isomorphic(cls, G1, G2):
         return nx.algorithms.isomorphism.isomorph.is_isomorphic(
             G1, G2)
+
+
+class BaremetalModelRoot(nx.DiGraph, base.Model):
+
+    """Cluster graph for an Openstack cluster: Baremetal Cluster."""
+
+    def __init__(self, stale=False):
+        super(BaremetalModelRoot, self).__init__()
+        self.stale = stale
+
+    def __nonzero__(self):
+        return not self.stale
+
+    __bool__ = __nonzero__
+
+    @staticmethod
+    def assert_node(obj):
+        if not isinstance(obj, element.IronicNode):
+            raise exception.IllegalArgumentException(
+                message=_("'obj' argument type is not valid: %s") % type(obj))
+
+    @lockutils.synchronized("baremetal_model")
+    def add_node(self, node):
+        self.assert_node(node)
+        super(BaremetalModelRoot, self).add_node(node.uuid, node)
+
+    @lockutils.synchronized("baremetal_model")
+    def remove_node(self, node):
+        self.assert_node(node)
+        try:
+            super(BaremetalModelRoot, self).remove_node(node.uuid)
+        except nx.NetworkXError as exc:
+            LOG.exception(exc)
+            raise exception.IronicNodeNotFound(name=node.uuid)
+
+    @lockutils.synchronized("baremetal_model")
+    def get_all_ironic_nodes(self):
+        return {uuid: cn for uuid, cn in self.nodes(data=True)
+                if isinstance(cn, element.IronicNode)}
+
+    @lockutils.synchronized("baremetal_model")
+    def get_node_by_uuid(self, uuid):
+        try:
+            return self._get_by_uuid(uuid)
+        except exception.BaremetalResourceNotFound:
+            raise exception.IronicNodeNotFound(name=uuid)
+
+    def _get_by_uuid(self, uuid):
+        try:
+            return self.node[uuid]
+        except Exception as exc:
+            LOG.exception(exc)
+        raise exception.BaremetalResourceNotFound(name=uuid)
+
+    def to_string(self):
+        return self.to_xml()
+
+    def to_xml(self):
+        root = etree.Element("ModelRoot")
+        # Build Ironic node tree
+        for cn in sorted(self.get_all_ironic_nodes().values(),
+                         key=lambda cn: cn.uuid):
+            ironic_node_el = cn.as_xml_element()
+            root.append(ironic_node_el)
+
+        return etree.tostring(root, pretty_print=True).decode('utf-8')
+
+    @classmethod
+    def from_xml(cls, data):
+        model = cls()
+
+        root = etree.fromstring(data)
+        for cn in root.findall('.//IronicNode'):
+            node = element.IronicNode(**cn.attrib)
+            model.add_node(node)
+
+        return model
+
+    @classmethod
+    def is_isomorphic(cls, G1, G2):
+        return nx.algorithms.isomorphism.isomorph.is_isomorphic(
+            G1, G2)

watcher/decision_engine/scope/baremetal.py

@@ -0,0 +1,46 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2018 ZTE Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_log import log
+
+from watcher.decision_engine.scope import base
+
+
+LOG = log.getLogger(__name__)
+
+
+class BaremetalScope(base.BaseScope):
+    """Baremetal Audit Scope Handler"""
+
+    def __init__(self, scope, config, osc=None):
+        super(BaremetalScope, self).__init__(scope, config)
+        self._osc = osc
+
+    def get_scoped_model(self, cluster_model):
+        """Leave only nodes and instances proposed in the audit scope"""
+        if not cluster_model:
+            return None
+
+        for scope in self.scope:
+            baremetal_scope = scope.get('baremetal')
+
+        if not baremetal_scope:
+            return cluster_model
+
+        # TODO(yumeng-bao): currently self.scope is always []
+        # Audit scoper for baremetal data model will be implemented:
+        # https://blueprints.launchpad.net/watcher/+spec/audit-scoper-for-baremetal-data-model
+        return cluster_model

watcher/decision_engine/strategy/strategies/base.py

@@ -85,6 +85,7 @@ class BaseStrategy(loadable.Loadable):
         self._collector_manager = None
         self._compute_model = None
         self._storage_model = None
+        self._baremetal_model = None
         self._input_parameters = utils.Struct()
         self._audit_scope = None
         self._datasource_backend = None
@@ -219,6 +220,29 @@ class BaseStrategy(loadable.Loadable):
 
         return self._storage_model
 
+    @property
+    def baremetal_model(self):
+        """Cluster data model
+
+        :returns: Cluster data model the strategy is executed on
+        :rtype model: :py:class:`~.ModelRoot` instance
+        """
+        if self._baremetal_model is None:
+            collector = self.collector_manager.get_cluster_model_collector(
+                'baremetal', osc=self.osc)
+            audit_scope_handler = collector.get_audit_scope_handler(
+                audit_scope=self.audit_scope)
+            self._baremetal_model = audit_scope_handler.get_scoped_model(
+                collector.get_latest_cluster_data_model())
+
+        if not self._baremetal_model:
+            raise exception.ClusterStateNotDefined()
+
+        if self._baremetal_model.stale:
+            raise exception.ClusterStateStale()
+
+        return self._baremetal_model
+
     @classmethod
     def get_schema(cls):
         """Defines a Schema that the input parameters shall comply to

watcher/tests/common/test_ironic_helper.py

@@ -0,0 +1,63 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2017 ZTE Corporation
+#
+# Authors:Yumeng Bao <bao.yumeng@zte.com.cn>
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import mock
+
+from watcher.common import clients
+from watcher.common import exception
+from watcher.common import ironic_helper
+from watcher.common import utils as w_utils
+from watcher.tests import base
+
+
+class TestIronicHelper(base.TestCase):
+
+    def setUp(self):
+        super(TestIronicHelper, self).setUp()
+
+        osc = clients.OpenStackClients()
+        p_ironic = mock.patch.object(osc, 'ironic')
+        p_ironic.start()
+        self.addCleanup(p_ironic.stop)
+        self.ironic_util = ironic_helper.IronicHelper(osc=osc)
+
+    @staticmethod
+    def fake_ironic_node():
+        node = mock.MagicMock()
+        node.uuid = w_utils.generate_uuid()
+        return node
+
+    def test_get_ironic_node_list(self):
+        node1 = self.fake_ironic_node()
+        self.ironic_util.ironic.node.list.return_value = [node1]
+        rt_nodes = self.ironic_util.get_ironic_node_list()
+        self.assertEqual(rt_nodes, [node1])
+
+    def test_get_ironic_node_by_uuid_success(self):
+        node1 = self.fake_ironic_node()
+        self.ironic_util.ironic.node.get.return_value = node1
+        node = self.ironic_util.get_ironic_node_by_uuid(node1.uuid)
+        self.assertEqual(node, node1)
+
+    def test_get_ironic_node_by_uuid_failure(self):
+        self.ironic_util.ironic.node.get.return_value = None
+        self.assertRaisesRegex(
+            exception.IronicNodeNotFound,
+            "The ironic node node1 could not be found",
+            self.ironic_util.get_ironic_node_by_uuid, 'node1')

watcher/tests/decision_engine/model/data/ironic_scenario_1.xml

@@ -0,0 +1,12 @@
+<ModelRoot>
+  <IronicNode uuid="c5941348-5a87-4016-94d4-4f9e0ce2b87a" power_state="power on" maintenance="false" maintenance_reason="null">
+    <extra>
+      <compute_node_id> 1</compute_node_id>
+    </extra>
+  </IronicNode>
+  <IronicNode uuid="c5941348-5a87-4016-94d4-4f9e0ce2b87c" power_state="power on" maintenance="false" maintenance_reason="null">
+    <extra>
+      <compute_node_id> 2</compute_node_id>
+    </extra>
+  </IronicNode>
+</ModelRoot>

watcher/tests/decision_engine/model/faker_cluster_state.py

@@ -20,6 +20,7 @@ import os
 
 import mock
 
+from watcher.common import utils
 from watcher.decision_engine.model.collector import base
 from watcher.decision_engine.model import element
 from watcher.decision_engine.model import model_root as modelroot
@@ -261,3 +262,55 @@ class FakerStorageModelCollector(base.BaseClusterDataModelCollector):
 
     def generate_scenario_1(self):
         return self.load_model('storage_scenario_1.xml')
+
+
+class FakerBaremetalModelCollector(base.BaseClusterDataModelCollector):
+
+    def __init__(self, config=None, osc=None):
+        if config is None:
+            config = mock.Mock(period=777)
+        super(FakerBaremetalModelCollector, self).__init__(config)
+
+    @property
+    def notification_endpoints(self):
+        return []
+
+    def get_audit_scope_handler(self, audit_scope):
+        return None
+
+    def load_data(self, filename):
+        cwd = os.path.abspath(os.path.dirname(__file__))
+        data_folder = os.path.join(cwd, "data")
+
+        with open(os.path.join(data_folder, filename), 'rb') as xml_file:
+            xml_data = xml_file.read()
+
+        return xml_data
+
+    def load_model(self, filename):
+        return modelroot.BaremetalModelRoot.from_xml(self.load_data(filename))
+
+    def execute(self):
+        return self._cluster_data_model or self.build_scenario_1()
+
+    def build_scenario_1(self):
+        model = modelroot.BaremetalModelRoot()
+        # number of nodes
+        node_count = 2
+
+        for i in range(0, node_count):
+            uuid = utils.generate_uuid()
+            node_attributes = {
+                "uuid": uuid,
+                "power_state": "power on",
+                "maintenance": "false",
+                "maintenance_reason": "null",
+                "extra": {"compute_node_id": i}
+            }
+            node = element.IronicNode(**node_attributes)
+            model.add_node(node)
+
+        return model
+
+    def generate_scenario_1(self):
+        return self.load_model('ironic_scenario_1.xml')

watcher/tests/decision_engine/model/test_element.py

@@ -152,3 +152,30 @@ class TestStorageElement(base.TestCase):
     def test_as_xml_element(self):
         el = self.cls(**self.data)
         el.as_xml_element()
+
+
+class TestIronicElement(base.TestCase):
+
+    scenarios = [
+        ("IronicNode_with_all_fields", dict(
+            cls=element.IronicNode,
+            data={
+                "uuid": 'FAKE_UUID',
+                "power_state": 'up',
+                "maintenance": "false",
+                "maintenance_reason": "null",
+                "extra": {"compute_node_id": 1}
+            })),
+        ("IronicNode_with_some_fields", dict(
+            cls=element.IronicNode,
+            data={
+                "uuid": 'FAKE_UUID',
+                "power_state": 'up',
+                "maintenance": "false",
+                "extra": {"compute_node_id": 1}
+            })),
+        ]
+
+    def test_as_xml_element(self):
+        el = self.cls(**self.data)
+        el.as_xml_element()

watcher/tests/decision_engine/model/test_model.py

@@ -367,3 +367,73 @@ class TestStorageModel(base.TestCase):
         self.assertEqual(volume, model.get_volume_by_uuid(uuid_))
         model.map_volume(volume, pool)
         self.assertEqual([volume], model.get_pool_volumes(pool))
+
+
+class TestBaremetalModel(base.TestCase):
+
+    def load_data(self, filename):
+        cwd = os.path.abspath(os.path.dirname(__file__))
+        data_folder = os.path.join(cwd, "data")
+
+        with open(os.path.join(data_folder, filename), 'rb') as xml_file:
+            xml_data = xml_file.read()
+
+        return xml_data
+
+    def load_model(self, filename):
+        return model_root.StorageModelRoot.from_xml(self.load_data(filename))
+
+    def test_model_structure(self):
+        fake_cluster = faker_cluster_state.FakerBaremetalModelCollector()
+        model1 = fake_cluster.build_scenario_1()
+        self.assertEqual(2, len(model1.get_all_ironic_nodes()))
+
+        expected_struct_str = self.load_data('ironic_scenario_1.xml')
+        model2 = model_root.BaremetalModelRoot.from_xml(expected_struct_str)
+        self.assertTrue(
+            model_root.BaremetalModelRoot.is_isomorphic(model2, model1))
+
+    def test_build_model_from_xml(self):
+        fake_cluster = faker_cluster_state.FakerBaremetalModelCollector()
+
+        expected_model = fake_cluster.generate_scenario_1()
+        struct_str = self.load_data('ironic_scenario_1.xml')
+
+        model = model_root.BaremetalModelRoot.from_xml(struct_str)
+        self.assertEqual(expected_model.to_string(), model.to_string())
+
+    def test_assert_node_raise(self):
+        model = model_root.BaremetalModelRoot()
+        node_uuid = uuidutils.generate_uuid()
+        node = element.IronicNode(uuid=node_uuid)
+        model.add_node(node)
+        self.assertRaises(exception.IllegalArgumentException,
+                          model.assert_node, "obj")
+
+    def test_add_node(self):
+        model = model_root.BaremetalModelRoot()
+        node_uuid = uuidutils.generate_uuid()
+        node = element.IronicNode(uuid=node_uuid)
+        model.add_node(node)
+        self.assertEqual(node, model.get_node_by_uuid(node_uuid))
+
+    def test_remove_node(self):
+        model = model_root.BaremetalModelRoot()
+        node_uuid = uuidutils.generate_uuid()
+        node = element.IronicNode(uuid=node_uuid)
+        model.add_node(node)
+        self.assertEqual(node, model.get_node_by_uuid(node_uuid))
+        model.remove_node(node)
+        self.assertRaises(exception.IronicNodeNotFound,
+                          model.get_node_by_uuid, node_uuid)
+
+    def test_get_all_ironic_nodes(self):
+        model = model_root.BaremetalModelRoot()
+        for i in range(10):
+            node_uuid = uuidutils.generate_uuid()
+            node = element.IronicNode(uuid=node_uuid)
+            model.add_node(node)
+        all_nodes = model.get_all_ironic_nodes()
+        for node_uuid in all_nodes:
+            node = model.get_node_by_uuid(node_uuid)
+            model.assert_node(node)