From 2ce96c7f9de045c5337769bbba326e832bf487f7 Mon Sep 17 00:00:00 2001 From: Alexey Weyl Date: Thu, 25 Aug 2016 14:50:27 +0300 Subject: [PATCH] Adding heat datasource Change-Id: I659b3517389a2862c936fecfae783e6d8f153009 --- devstack/README.rst | 11 +- .../datasources_values/heat.stack.yaml | 46 +++++ requirements.txt | 1 + test-requirements.txt | 1 + vitrage/clients.py | 23 ++- vitrage/common/constants.py | 1 + vitrage/datasources/heat/__init__.py | 15 ++ vitrage/datasources/heat/stack/__init__.py | 38 +++++ vitrage/datasources/heat/stack/driver.py | 117 +++++++++++++ vitrage/datasources/heat/stack/transformer.py | 158 ++++++++++++++++++ vitrage/datasources/zabbix/driver.py | 2 +- 11 files changed, 409 insertions(+), 4 deletions(-) create mode 100644 etc/vitrage/datasources_values/heat.stack.yaml create mode 100644 vitrage/datasources/heat/__init__.py create mode 100644 vitrage/datasources/heat/stack/__init__.py create mode 100644 vitrage/datasources/heat/stack/driver.py create mode 100644 vitrage/datasources/heat/stack/transformer.py diff --git a/devstack/README.rst b/devstack/README.rst index 2550f5ecd..ab13d76f3 100644 --- a/devstack/README.rst +++ b/devstack/README.rst @@ -40,4 +40,13 @@ Enabling Vitrage in DevStack notification_topics = notifications,vitrage_notifications notification_driver=messagingv2 -6. Run ``stack.sh``. +6. Add this to add notification from heat to vitrage + +:: + + [[post-config|$HEAT_CONF]] + [DEFAULT] + notification_topics = notifications,vitrage_notifications + notification_driver=messagingv2 + +7. Run ``stack.sh``. diff --git a/etc/vitrage/datasources_values/heat.stack.yaml b/etc/vitrage/datasources_values/heat.stack.yaml new file mode 100644 index 000000000..a111355e7 --- /dev/null +++ b/etc/vitrage/datasources_values/heat.stack.yaml @@ -0,0 +1,46 @@ +category: RESOURCE +values: + - aggregated values: + priority: 40 + original values: + - name: DELETE FAILED + operational_value: ERROR + - name: CREATE FAILED + operational_value: ERROR + - name: UPDATE FAILED + operational_value: ERROR + - name: RESUME FAILED + operational_value: ERROR + - name: SUSPEND FAILED + operational_value: ERROR + - aggregated values: + priority: 30 + original values: + - name: DELETE IN PROGRESS + operational_value: TRANSIENT + - name: CREATE IN PROGRESS + operational_value: TRANSIENT + - name: UPDATE IN PROGRESS + operational_value: TRANSIENT + - name: RESUME IN PROGRESS + operational_value: TRANSIENT + - name: SUSPEND IN PROGRESS + operational_value: TRANSIENT + - aggregated values: + priority: 20 + original values: + - name: SUBOPTIMAL + operational_value: SUBOPTIMAL + - aggregated values: + priority: 10 + original values: + - name: DELETE COMPLETE + operational_value: OK + - name: CREATE COMPLETE + operational_value: OK + - name: UPDATE COMPLETE + operational_value: OK + - name: RESUME COMPLETE + operational_value: OK + - name: SUSPEND COMPLETE + operational_value: OK diff --git a/requirements.txt b/requirements.txt index 3ce8d6399..e18de129a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ python-dateutil>=2.4.2 python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0 python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0 python-novaclient>=2.26.0 +python-heatclient>=1.1.0 # Apache-2.0 pyzabbix>=0.7.4 # LGPL networkx>=1.10 oslo.config>=2.7.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 219bf8c26..1250d68e3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,6 +11,7 @@ python-ceilometerclient>=2.2.1 # Apache-2.0 python-cinderclient>=1.3.1 # Apache-2.0 python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0 python-novaclient>=2.26.0 +python-heatclient>=1.1.0 # Apache-2.0 python-subunit>=0.0.18 pyzabbix>=0.7.4 # LGPL sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 diff --git a/vitrage/clients.py b/vitrage/clients.py index 80396e079..8dc3ba557 100644 --- a/vitrage/clients.py +++ b/vitrage/clients.py @@ -12,15 +12,16 @@ # License for the specific language governing permissions and limitations # under the License. +import keystoneauth1.identity.v2 as v2 +import keystoneauth1.session as kssession from oslo_config import cfg from oslo_log import log from ceilometerclient import client as cm_client from cinderclient import client as cin_client +from heatclient.v1 import client as he_client from neutronclient.v2_0 import client as ne_client from novaclient import client as n_client - - from vitrage import keystone_client LOG = log.getLogger(__name__) @@ -29,6 +30,7 @@ OPTS = [ cfg.StrOpt('aodh_version', default='2', help='Aodh version'), cfg.FloatOpt('nova_version', default='2.11', help='Nova version'), cfg.StrOpt('cinder_version', default='2', help='Cinder version'), + cfg.StrOpt('heat_version', default='1', help='Heat version'), ] @@ -93,3 +95,20 @@ def neutron_client(conf): return client except Exception as e: LOG.exception('Create Neutron client - Got Exception: %s', e) + + +def heat_client(conf): + """Get an instance of heat client""" + # auth_config = conf.service_credentials + try: + auth = v2.Password( + auth_url=conf.service_credentials.auth_url + '/v2.0', + username=conf.service_credentials.username, + password=conf.service_credentials.password, + tenant_name=conf.service_credentials.project_name) + session = kssession.Session(auth=auth) + endpoint = session.get_endpoint(service_type='orchestration', + interface='publicURL') + return he_client.Client(session=session, endpoint=endpoint) + except Exception as e: + LOG.exception('Create Heat client - Got Exception: %s', e) diff --git a/vitrage/common/constants.py b/vitrage/common/constants.py index 281bc36d7..a99821f58 100644 --- a/vitrage/common/constants.py +++ b/vitrage/common/constants.py @@ -55,6 +55,7 @@ class EdgeLabel(object): ATTACHED_PRIVATE = 'attached_private' CONNECT = 'connect' MANAGED_BY = 'managed_by' + COMPRISED = 'comprised' edge_labels = [EdgeLabel.ON, EdgeLabel.CONTAINS, diff --git a/vitrage/datasources/heat/__init__.py b/vitrage/datasources/heat/__init__.py new file mode 100644 index 000000000..dd32b852f --- /dev/null +++ b/vitrage/datasources/heat/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/datasources/heat/stack/__init__.py b/vitrage/datasources/heat/stack/__init__.py new file mode 100644 index 000000000..501943b9e --- /dev/null +++ b/vitrage/datasources/heat/stack/__init__.py @@ -0,0 +1,38 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from vitrage.common.constants import UpdateMethod + +HEAT_STACK_DATASOURCE = 'heat.stack' + +OPTS = [ + cfg.StrOpt('transformer', + default='vitrage.datasources.heat.stack.transformer.' + 'HeatStackTransformer', + help='Heat stack transformer class path', + required=True), + cfg.StrOpt('driver', + default='vitrage.datasources.heat.stack.driver.' + 'HeatStackDriver', + help='Heat stack driver class path', + required=True), + cfg.StrOpt('update_method', + default=UpdateMethod.PUSH, + help='None: updates only via Vitrage periodic snapshots.' + 'Pull: updates every [changes_interval] seconds.' + 'Push: updates by getting notifications from the' + ' datasource itself.', + required=True), +] diff --git a/vitrage/datasources/heat/stack/driver.py b/vitrage/datasources/heat/stack/driver.py new file mode 100644 index 000000000..85bba2a38 --- /dev/null +++ b/vitrage/datasources/heat/stack/driver.py @@ -0,0 +1,117 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from vitrage import clients +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import SyncMode +from vitrage.datasources.cinder.volume import CINDER_VOLUME_DATASOURCE +from vitrage.datasources.driver_base import DriverBase +from vitrage.datasources.heat.stack import HEAT_STACK_DATASOURCE +from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE +from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE +from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE + +LOG = logging.getLogger(__name__) + + +class HeatStackDriver(DriverBase): + + _client = None + + FAILED = 'FAILED' + + RESOURCE_TYPE_CONVERSION = { + 'OS::Nova::Server': NOVA_INSTANCE_DATASOURCE, + 'OS::Cinder::Volume': CINDER_VOLUME_DATASOURCE, + 'OS::Neutron::Net': NEUTRON_NETWORK_DATASOURCE, + 'OS::Neutron::Port': NEUTRON_PORT_DATASOURCE + } + + def __init__(self, conf): + super(HeatStackDriver, self).__init__() + self.conf = conf + self._filter_resource_types() + HeatStackDriver._client = self.client + + @property + def client(self): + if not self._client: + self._client = clients.heat_client(self.conf) + return self._client + + @staticmethod + def get_topic(conf): + return conf[HEAT_STACK_DATASOURCE].notification_topic + + @staticmethod + def get_event_types(conf): + return ['orchestration.stack.create.end', + 'orchestration.stack.delete.end', + 'orchestration.stack.update.error', + 'orchestration.stack.update.end', + 'orchestration.stack.suspend.error', + 'orchestration.stack.suspend.end', + 'orchestration.stack.resume.error', + 'orchestration.stack.resume.end'] + + @staticmethod + def enrich_event(event, event_type): + # TODO(Nofar): add call to get resources of the stack if not deleted + # change transformer that if delete we remove the stack from the graph + # and hence all the edges to it + + event[DSProps.EVENT_TYPE] = event_type + event = HeatStackDriver._retrieve_stack_resources( + event, event['stack_identity']) + + return HeatStackDriver.make_pickleable([event], + HEAT_STACK_DATASOURCE, + SyncMode.UPDATE)[0] + + def _filter_resource_types(self): + types = self.conf.datasources.types + tmp_dict = {} + + for key, value in HeatStackDriver.RESOURCE_TYPE_CONVERSION.items(): + if value in types: + tmp_dict[key] = value + + HeatStackDriver.RESOURCE_TYPE_CONVERSION = tmp_dict + + def _make_stacks_list(self, stacks): + return [stack.__dict__ for stack in stacks + if self.FAILED not in stack.__dict__['stack_status']] + + def _append_stacks_resources(self, stacks): + return [self._retrieve_stack_resources(stack, stack['id']) + for stack in stacks] + + @classmethod + def _retrieve_stack_resources(cls, stack, stack_id): + resources = cls._client.resources.list(stack_id) + stack['resources'] = [resource.__dict__ for resource in resources + if resource.__dict__['resource_type'] in + HeatStackDriver.RESOURCE_TYPE_CONVERSION] + return stack + + def get_all(self, sync_mode): + stacks = self.client.stacks.list(global_tenant=True) + stacks_list = self._make_stacks_list(stacks) + stacks_with_resources = self._append_stacks_resources(stacks_list) + return self.make_pickleable(stacks_with_resources, + HEAT_STACK_DATASOURCE, + sync_mode, + 'manager') diff --git a/vitrage/datasources/heat/stack/transformer.py b/vitrage/datasources/heat/stack/transformer.py new file mode 100644 index 000000000..bee07fc9e --- /dev/null +++ b/vitrage/datasources/heat/stack/transformer.py @@ -0,0 +1,158 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import EdgeLabel +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import EventAction +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.cinder.volume import CINDER_VOLUME_DATASOURCE +from vitrage.datasources.heat.stack import HEAT_STACK_DATASOURCE +from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE +from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE +from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE +from vitrage.datasources.resource_transformer_base import \ + ResourceTransformerBase +from vitrage.datasources import transformer_base as tbase +from vitrage.datasources.transformer_base import build_key +from vitrage.datasources.transformer_base import extract_field_value +from vitrage.datasources.transformer_base import Neighbor +import vitrage.graph.utils as graph_utils + + +LOG = logging.getLogger(__name__) + + +class HeatStackTransformer(ResourceTransformerBase): + + RESOURCE_TYPE_CONVERSION = { + 'OS::Nova::Server': NOVA_INSTANCE_DATASOURCE, + 'OS::Cinder::Volume': CINDER_VOLUME_DATASOURCE, + 'OS::Neutron::Net': NEUTRON_NETWORK_DATASOURCE, + 'OS::Neutron::Port': NEUTRON_PORT_DATASOURCE + } + + # Event types which need to refer them differently + UPDATE_EVENT_TYPES = { + 'orchestration.stack.delete.end': EventAction.DELETE_ENTITY, + } + + def __init__(self, transformers, conf): + super(HeatStackTransformer, self).__init__(transformers, conf) + + def _create_snapshot_entity_vertex(self, entity_event): + stack_name = extract_field_value(entity_event, 'stack_name') + stack_id = extract_field_value(entity_event, 'id') + stack_state = extract_field_value(entity_event, 'stack_status') + timestamp = extract_field_value(entity_event, 'creation_time') + project_id = extract_field_value(entity_event, 'project_id') + + return self._create_vertex(entity_event, + stack_name, + stack_id, + stack_state, + timestamp, + project_id) + + def _create_update_entity_vertex(self, entity_event): + + volume_name = extract_field_value(entity_event, 'stack_name') + volume_id = extract_field_value(entity_event, 'stack_identity') + volume_state = extract_field_value(entity_event, 'state') + timestamp = entity_event.get('create_at', None) + project_id = entity_event.get('tenant_id', None) + + return self._create_vertex(entity_event, + volume_name, + volume_id, + volume_state, + timestamp, + project_id) + + def _create_vertex(self, + entity_event, + stack_name, + stack_id, + stack_state, + update_timestamp, + project_id): + metadata = { + VProps.NAME: stack_name, + VProps.PROJECT_ID: project_id, + } + + entity_key = self._create_entity_key(entity_event) + + sample_timestamp = entity_event[DSProps.SAMPLE_DATE] + + return graph_utils.create_vertex( + entity_key, + entity_id=stack_id, + entity_category=EntityCategory.RESOURCE, + entity_type=HEAT_STACK_DATASOURCE, + entity_state=stack_state, + sample_timestamp=sample_timestamp, + update_timestamp=update_timestamp, + metadata=metadata) + + def _create_snapshot_neighbors(self, entity_event): + return self._create_neighbors(entity_event) + + def _create_update_neighbors(self, entity_event): + return self._create_neighbors(entity_event) + + def _create_entity_key(self, entity_event): + + is_update_event = tbase.is_update_event(entity_event) + id_field_path = 'stack_identity' if is_update_event else 'id' + volume_id = extract_field_value(entity_event, id_field_path) + + key_fields = self._key_values(HEAT_STACK_DATASOURCE, volume_id) + return build_key(key_fields) + + def _create_neighbors(self, entity_event): + return [self._create_neighbor(entity_event, neighbor) + for neighbor in entity_event['resources']] + + def _create_neighbor(self, + entity_event, + neighbor): + datasource_type = \ + self.RESOURCE_TYPE_CONVERSION[neighbor['resource_type']] + transformer = self.transformers.get(datasource_type, None) + + stack_vitrage_id = self._create_entity_key(entity_event) + + neighbor_id = neighbor['physical_resource_id'] + + sample_timestamp = entity_event[DSProps.SAMPLE_DATE] + + properties = { + VProps.ID: neighbor_id, + VProps.TYPE: datasource_type, + VProps.SAMPLE_TIMESTAMP: sample_timestamp + } + instance_vertex = transformer.create_placeholder_vertex(**properties) + + relationship_edge = graph_utils.create_edge( + source_id=stack_vitrage_id, + target_id=instance_vertex.vertex_id, + relationship_type=EdgeLabel.COMPRISED) + + return Neighbor(instance_vertex, relationship_edge) + + def get_type(self): + return HEAT_STACK_DATASOURCE diff --git a/vitrage/datasources/zabbix/driver.py b/vitrage/datasources/zabbix/driver.py index 320a315f8..b82916d27 100644 --- a/vitrage/datasources/zabbix/driver.py +++ b/vitrage/datasources/zabbix/driver.py @@ -38,7 +38,7 @@ class ZabbixDriver(AlarmDriverBase): def __init__(self, conf): super(ZabbixDriver, self).__init__() self.conf = conf - if ZabbixDriver.conf_map is None: + if not ZabbixDriver.conf_map: ZabbixDriver.conf_map =\ ZabbixDriver._configuration_mapping(conf) self._client = None