Adding heat datasource

Change-Id: I659b3517389a2862c936fecfae783e6d8f153009

devstack/README.rst

@@ -40,4 +40,13 @@ Enabling Vitrage in DevStack
    notification_topics = notifications,vitrage_notifications
    notification_driver=messagingv2
 
-6. Run ``stack.sh``.
+6. Add this to add notification from heat to vitrage
+
+::
+
+   [[post-config|$HEAT_CONF]]
+   [DEFAULT]
+   notification_topics = notifications,vitrage_notifications
+   notification_driver=messagingv2
+
+7. Run ``stack.sh``.

etc/vitrage/datasources_values/heat.stack.yaml

@@ -0,0 +1,46 @@
+category: RESOURCE
+values:
+  - aggregated values:
+      priority: 40
+      original values:
+        - name: DELETE FAILED
+          operational_value: ERROR
+        - name: CREATE FAILED
+          operational_value: ERROR
+        - name: UPDATE FAILED
+          operational_value: ERROR
+        - name: RESUME FAILED
+          operational_value: ERROR
+        - name: SUSPEND FAILED
+          operational_value: ERROR
+  - aggregated values:
+      priority: 30
+      original values:
+        - name: DELETE IN PROGRESS
+          operational_value: TRANSIENT
+        - name: CREATE IN PROGRESS
+          operational_value: TRANSIENT
+        - name: UPDATE IN PROGRESS
+          operational_value: TRANSIENT
+        - name: RESUME IN PROGRESS
+          operational_value: TRANSIENT
+        - name: SUSPEND IN PROGRESS
+          operational_value: TRANSIENT
+  - aggregated values:
+      priority: 20
+      original values:
+        - name: SUBOPTIMAL
+          operational_value: SUBOPTIMAL
+  - aggregated values:
+      priority: 10
+      original values:
+        - name: DELETE COMPLETE
+          operational_value: OK
+        - name: CREATE COMPLETE
+          operational_value: OK
+        - name: UPDATE COMPLETE
+          operational_value: OK
+        - name: RESUME COMPLETE
+          operational_value: OK
+        - name: SUSPEND COMPLETE
+          operational_value: OK

requirements.txt

@@ -11,6 +11,7 @@ python-dateutil>=2.4.2
 python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0
 python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0
 python-novaclient>=2.26.0
+python-heatclient>=1.1.0 # Apache-2.0
 pyzabbix>=0.7.4  # LGPL
 networkx>=1.10
 oslo.config>=2.7.0 # Apache-2.0

test-requirements.txt

@@ -11,6 +11,7 @@ python-ceilometerclient>=2.2.1  # Apache-2.0
 python-cinderclient>=1.3.1 # Apache-2.0
 python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0
 python-novaclient>=2.26.0
+python-heatclient>=1.1.0 # Apache-2.0
 python-subunit>=0.0.18
 pyzabbix>=0.7.4  # LGPL
 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2

vitrage/clients.py

@@ -12,15 +12,16 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import keystoneauth1.identity.v2 as v2
+import keystoneauth1.session as kssession
 from oslo_config import cfg
 from oslo_log import log
 
 from ceilometerclient import client as cm_client
 from cinderclient import client as cin_client
+from heatclient.v1 import client as he_client
 from neutronclient.v2_0 import client as ne_client
 from novaclient import client as n_client
-
-
 from vitrage import keystone_client
 
 LOG = log.getLogger(__name__)
@@ -29,6 +30,7 @@ OPTS = [
     cfg.StrOpt('aodh_version', default='2', help='Aodh version'),
     cfg.FloatOpt('nova_version', default='2.11', help='Nova version'),
     cfg.StrOpt('cinder_version', default='2', help='Cinder version'),
+    cfg.StrOpt('heat_version', default='1', help='Heat version'),
 ]
 
 
@@ -93,3 +95,20 @@ def neutron_client(conf):
         return client
     except Exception as e:
         LOG.exception('Create Neutron client - Got Exception: %s', e)
+
+
+def heat_client(conf):
+    """Get an instance of heat client"""
+    # auth_config = conf.service_credentials
+    try:
+        auth = v2.Password(
+            auth_url=conf.service_credentials.auth_url + '/v2.0',
+            username=conf.service_credentials.username,
+            password=conf.service_credentials.password,
+            tenant_name=conf.service_credentials.project_name)
+        session = kssession.Session(auth=auth)
+        endpoint = session.get_endpoint(service_type='orchestration',
+                                        interface='publicURL')
+        return he_client.Client(session=session, endpoint=endpoint)
+    except Exception as e:
+        LOG.exception('Create Heat client - Got Exception: %s', e)

vitrage/common/constants.py

@@ -55,6 +55,7 @@ class EdgeLabel(object):
     ATTACHED_PRIVATE = 'attached_private'
     CONNECT = 'connect'
     MANAGED_BY = 'managed_by'
+    COMPRISED = 'comprised'
 
 edge_labels = [EdgeLabel.ON,
                EdgeLabel.CONTAINS,

vitrage/datasources/heat/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2016 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__author__ = 'stack'

vitrage/datasources/heat/stack/__init__.py

@@ -0,0 +1,38 @@
+# Copyright 2016 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR  CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from vitrage.common.constants import UpdateMethod
+
+HEAT_STACK_DATASOURCE = 'heat.stack'
+
+OPTS = [
+    cfg.StrOpt('transformer',
+               default='vitrage.datasources.heat.stack.transformer.'
+                       'HeatStackTransformer',
+               help='Heat stack transformer class path',
+               required=True),
+    cfg.StrOpt('driver',
+               default='vitrage.datasources.heat.stack.driver.'
+                       'HeatStackDriver',
+               help='Heat stack driver class path',
+               required=True),
+    cfg.StrOpt('update_method',
+               default=UpdateMethod.PUSH,
+               help='None: updates only via Vitrage periodic snapshots.'
+                    'Pull: updates every [changes_interval] seconds.'
+                    'Push: updates by getting notifications from the'
+                    ' datasource itself.',
+               required=True),
+]

vitrage/datasources/heat/stack/driver.py

@@ -0,0 +1,117 @@
+# Copyright 2016 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log as logging
+
+from vitrage import clients
+from vitrage.common.constants import DatasourceProperties as DSProps
+from vitrage.common.constants import SyncMode
+from vitrage.datasources.cinder.volume import CINDER_VOLUME_DATASOURCE
+from vitrage.datasources.driver_base import DriverBase
+from vitrage.datasources.heat.stack import HEAT_STACK_DATASOURCE
+from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE
+from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE
+from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE
+
+LOG = logging.getLogger(__name__)
+
+
+class HeatStackDriver(DriverBase):
+
+    _client = None
+
+    FAILED = 'FAILED'
+
+    RESOURCE_TYPE_CONVERSION = {
+        'OS::Nova::Server': NOVA_INSTANCE_DATASOURCE,
+        'OS::Cinder::Volume': CINDER_VOLUME_DATASOURCE,
+        'OS::Neutron::Net': NEUTRON_NETWORK_DATASOURCE,
+        'OS::Neutron::Port': NEUTRON_PORT_DATASOURCE
+    }
+
+    def __init__(self, conf):
+        super(HeatStackDriver, self).__init__()
+        self.conf = conf
+        self._filter_resource_types()
+        HeatStackDriver._client = self.client
+
+    @property
+    def client(self):
+        if not self._client:
+            self._client = clients.heat_client(self.conf)
+        return self._client
+
+    @staticmethod
+    def get_topic(conf):
+        return conf[HEAT_STACK_DATASOURCE].notification_topic
+
+    @staticmethod
+    def get_event_types(conf):
+        return ['orchestration.stack.create.end',
+                'orchestration.stack.delete.end',
+                'orchestration.stack.update.error',
+                'orchestration.stack.update.end',
+                'orchestration.stack.suspend.error',
+                'orchestration.stack.suspend.end',
+                'orchestration.stack.resume.error',
+                'orchestration.stack.resume.end']
+
+    @staticmethod
+    def enrich_event(event, event_type):
+        # TODO(Nofar): add call to get resources of the stack if not deleted
+        # change transformer that if delete we remove the stack from the graph
+        # and hence all the edges to it
+
+        event[DSProps.EVENT_TYPE] = event_type
+        event = HeatStackDriver._retrieve_stack_resources(
+            event, event['stack_identity'])
+
+        return HeatStackDriver.make_pickleable([event],
+                                               HEAT_STACK_DATASOURCE,
+                                               SyncMode.UPDATE)[0]
+
+    def _filter_resource_types(self):
+        types = self.conf.datasources.types
+        tmp_dict = {}
+
+        for key, value in HeatStackDriver.RESOURCE_TYPE_CONVERSION.items():
+            if value in types:
+                tmp_dict[key] = value
+
+        HeatStackDriver.RESOURCE_TYPE_CONVERSION = tmp_dict
+
+    def _make_stacks_list(self, stacks):
+        return [stack.__dict__ for stack in stacks
+                if self.FAILED not in stack.__dict__['stack_status']]
+
+    def _append_stacks_resources(self, stacks):
+        return [self._retrieve_stack_resources(stack, stack['id'])
+                for stack in stacks]
+
+    @classmethod
+    def _retrieve_stack_resources(cls, stack, stack_id):
+        resources = cls._client.resources.list(stack_id)
+        stack['resources'] = [resource.__dict__ for resource in resources
+                              if resource.__dict__['resource_type'] in
+                              HeatStackDriver.RESOURCE_TYPE_CONVERSION]
+        return stack
+
+    def get_all(self, sync_mode):
+        stacks = self.client.stacks.list(global_tenant=True)
+        stacks_list = self._make_stacks_list(stacks)
+        stacks_with_resources = self._append_stacks_resources(stacks_list)
+        return self.make_pickleable(stacks_with_resources,
+                                    HEAT_STACK_DATASOURCE,
+                                    sync_mode,
+                                    'manager')

vitrage/datasources/heat/stack/transformer.py

@@ -0,0 +1,158 @@
+# Copyright 2016 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_log import log as logging
+
+from vitrage.common.constants import DatasourceProperties as DSProps
+from vitrage.common.constants import EdgeLabel
+from vitrage.common.constants import EntityCategory
+from vitrage.common.constants import EventAction
+from vitrage.common.constants import VertexProperties as VProps
+from vitrage.datasources.cinder.volume import CINDER_VOLUME_DATASOURCE
+from vitrage.datasources.heat.stack import HEAT_STACK_DATASOURCE
+from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE
+from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE
+from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE
+from vitrage.datasources.resource_transformer_base import \
+    ResourceTransformerBase
+from vitrage.datasources import transformer_base as tbase
+from vitrage.datasources.transformer_base import build_key
+from vitrage.datasources.transformer_base import extract_field_value
+from vitrage.datasources.transformer_base import Neighbor
+import vitrage.graph.utils as graph_utils
+
+
+LOG = logging.getLogger(__name__)
+
+
+class HeatStackTransformer(ResourceTransformerBase):
+
+    RESOURCE_TYPE_CONVERSION = {
+        'OS::Nova::Server': NOVA_INSTANCE_DATASOURCE,
+        'OS::Cinder::Volume': CINDER_VOLUME_DATASOURCE,
+        'OS::Neutron::Net': NEUTRON_NETWORK_DATASOURCE,
+        'OS::Neutron::Port': NEUTRON_PORT_DATASOURCE
+    }
+
+    # Event types which need to refer them differently
+    UPDATE_EVENT_TYPES = {
+        'orchestration.stack.delete.end': EventAction.DELETE_ENTITY,
+    }
+
+    def __init__(self, transformers, conf):
+        super(HeatStackTransformer, self).__init__(transformers, conf)
+
+    def _create_snapshot_entity_vertex(self, entity_event):
+        stack_name = extract_field_value(entity_event, 'stack_name')
+        stack_id = extract_field_value(entity_event, 'id')
+        stack_state = extract_field_value(entity_event, 'stack_status')
+        timestamp = extract_field_value(entity_event, 'creation_time')
+        project_id = extract_field_value(entity_event, 'project_id')
+
+        return self._create_vertex(entity_event,
+                                   stack_name,
+                                   stack_id,
+                                   stack_state,
+                                   timestamp,
+                                   project_id)
+
+    def _create_update_entity_vertex(self, entity_event):
+
+        volume_name = extract_field_value(entity_event, 'stack_name')
+        volume_id = extract_field_value(entity_event, 'stack_identity')
+        volume_state = extract_field_value(entity_event, 'state')
+        timestamp = entity_event.get('create_at', None)
+        project_id = entity_event.get('tenant_id', None)
+
+        return self._create_vertex(entity_event,
+                                   volume_name,
+                                   volume_id,
+                                   volume_state,
+                                   timestamp,
+                                   project_id)
+
+    def _create_vertex(self,
+                       entity_event,
+                       stack_name,
+                       stack_id,
+                       stack_state,
+                       update_timestamp,
+                       project_id):
+        metadata = {
+            VProps.NAME: stack_name,
+            VProps.PROJECT_ID: project_id,
+        }
+
+        entity_key = self._create_entity_key(entity_event)
+
+        sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
+
+        return graph_utils.create_vertex(
+            entity_key,
+            entity_id=stack_id,
+            entity_category=EntityCategory.RESOURCE,
+            entity_type=HEAT_STACK_DATASOURCE,
+            entity_state=stack_state,
+            sample_timestamp=sample_timestamp,
+            update_timestamp=update_timestamp,
+            metadata=metadata)
+
+    def _create_snapshot_neighbors(self, entity_event):
+        return self._create_neighbors(entity_event)
+
+    def _create_update_neighbors(self, entity_event):
+        return self._create_neighbors(entity_event)
+
+    def _create_entity_key(self, entity_event):
+
+        is_update_event = tbase.is_update_event(entity_event)
+        id_field_path = 'stack_identity' if is_update_event else 'id'
+        volume_id = extract_field_value(entity_event, id_field_path)
+
+        key_fields = self._key_values(HEAT_STACK_DATASOURCE, volume_id)
+        return build_key(key_fields)
+
+    def _create_neighbors(self, entity_event):
+        return [self._create_neighbor(entity_event, neighbor)
+                for neighbor in entity_event['resources']]
+
+    def _create_neighbor(self,
+                         entity_event,
+                         neighbor):
+        datasource_type = \
+            self.RESOURCE_TYPE_CONVERSION[neighbor['resource_type']]
+        transformer = self.transformers.get(datasource_type, None)
+
+        stack_vitrage_id = self._create_entity_key(entity_event)
+
+        neighbor_id = neighbor['physical_resource_id']
+
+        sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
+
+        properties = {
+            VProps.ID: neighbor_id,
+            VProps.TYPE: datasource_type,
+            VProps.SAMPLE_TIMESTAMP: sample_timestamp
+        }
+        instance_vertex = transformer.create_placeholder_vertex(**properties)
+
+        relationship_edge = graph_utils.create_edge(
+            source_id=stack_vitrage_id,
+            target_id=instance_vertex.vertex_id,
+            relationship_type=EdgeLabel.COMPRISED)
+
+        return Neighbor(instance_vertex, relationship_edge)
+
+    def get_type(self):
+        return HEAT_STACK_DATASOURCE

vitrage/datasources/zabbix/driver.py

@@ -38,7 +38,7 @@ class ZabbixDriver(AlarmDriverBase):
     def __init__(self, conf):
         super(ZabbixDriver, self).__init__()
         self.conf = conf
-        if ZabbixDriver.conf_map is None:
+        if not ZabbixDriver.conf_map:
             ZabbixDriver.conf_map =\
                 ZabbixDriver._configuration_mapping(conf)
         self._client = None