openstack/vitrage
Code Issues Proposed changes

Adding heat datasource

Browse Source 
Change-Id: I659b3517389a2862c936fecfae783e6d8f153009
This commit is contained in:
Alexey Weyl 2016-08-25 14:50:27 +03:00
parent e4541d9f52
commit 2ce96c7f9d
11 changed files with 409 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
devstack/README.rst
View File

@ -40,4 +40,13 @@ Enabling Vitrage in DevStack
notification_topics = notifications,vitrage_notifications
notification_driver=messagingv2
6. Run ``stack.sh``.
6. Add this to add notification from heat to vitrage
::
[[post-config|$HEAT_CONF]]
[DEFAULT]
notification_topics = notifications,vitrage_notifications
notification_driver=messagingv2
7. Run ``stack.sh``.

46
etc/vitrage/datasources_values/heat.stack.yaml Normal file
View File

@ -0,0 +1,46 @@
category: RESOURCE
values:
- aggregated values:
priority: 40
original values:
- name: DELETE FAILED
operational_value: ERROR
- name: CREATE FAILED
operational_value: ERROR
- name: UPDATE FAILED
operational_value: ERROR
- name: RESUME FAILED
operational_value: ERROR
- name: SUSPEND FAILED
operational_value: ERROR
- aggregated values:
priority: 30
original values:
- name: DELETE IN PROGRESS
operational_value: TRANSIENT
- name: CREATE IN PROGRESS
operational_value: TRANSIENT
- name: UPDATE IN PROGRESS
operational_value: TRANSIENT
- name: RESUME IN PROGRESS
operational_value: TRANSIENT
- name: SUSPEND IN PROGRESS
operational_value: TRANSIENT
- aggregated values:
priority: 20
original values:
- name: SUBOPTIMAL
operational_value: SUBOPTIMAL
- aggregated values:
priority: 10
original values:
- name: DELETE COMPLETE
operational_value: OK
- name: CREATE COMPLETE
operational_value: OK
- name: UPDATE COMPLETE
operational_value: OK
- name: RESUME COMPLETE
operational_value: OK
- name: SUSPEND COMPLETE
operational_value: OK

1
requirements.txt
View File

@ -11,6 +11,7 @@ python-dateutil>=2.4.2
python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0
python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0
python-novaclient>=2.26.0
python-heatclient>=1.1.0 # Apache-2.0
pyzabbix>=0.7.4 # LGPL
networkx>=1.10
oslo.config>=2.7.0 # Apache-2.0

1
test-requirements.txt
View File

@ -11,6 +11,7 @@ python-ceilometerclient>=2.2.1 # Apache-2.0
python-cinderclient>=1.3.1 # Apache-2.0
python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0
python-novaclient>=2.26.0
python-heatclient>=1.1.0 # Apache-2.0
python-subunit>=0.0.18
pyzabbix>=0.7.4 # LGPL
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2

23
vitrage/clients.py
View File

@ -12,15 +12,16 @@
# License for the specific language governing permissions and limitations
# under the License.
import keystoneauth1.identity.v2 as v2
import keystoneauth1.session as kssession
from oslo_config import cfg
from oslo_log import log
from ceilometerclient import client as cm_client
from cinderclient import client as cin_client
from heatclient.v1 import client as he_client
from neutronclient.v2_0 import client as ne_client
from novaclient import client as n_client
from vitrage import keystone_client
LOG = log.getLogger(__name__)
@ -29,6 +30,7 @@ OPTS = [
cfg.StrOpt('aodh_version', default='2', help='Aodh version'),
cfg.FloatOpt('nova_version', default='2.11', help='Nova version'),
cfg.StrOpt('cinder_version', default='2', help='Cinder version'),
cfg.StrOpt('heat_version', default='1', help='Heat version'),
]
@ -93,3 +95,20 @@ def neutron_client(conf):
return client
except Exception as e:
LOG.exception('Create Neutron client - Got Exception: %s', e)
def heat_client(conf):
"""Get an instance of heat client"""
# auth_config = conf.service_credentials
try:
auth = v2.Password(
auth_url=conf.service_credentials.auth_url + '/v2.0',
username=conf.service_credentials.username,
password=conf.service_credentials.password,
tenant_name=conf.service_credentials.project_name)
session = kssession.Session(auth=auth)
endpoint = session.get_endpoint(service_type='orchestration',
interface='publicURL')
return he_client.Client(session=session, endpoint=endpoint)
except Exception as e:
LOG.exception('Create Heat client - Got Exception: %s', e)

1
vitrage/common/constants.py
View File

@ -55,6 +55,7 @@ class EdgeLabel(object):
ATTACHED_PRIVATE = 'attached_private'
CONNECT = 'connect'
MANAGED_BY = 'managed_by'
COMPRISED = 'comprised'
edge_labels = [EdgeLabel.ON,
EdgeLabel.CONTAINS,

15
vitrage/datasources/heat/__init__.py Normal file
View File

@ -0,0 +1,15 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
__author__ = 'stack'

38
vitrage/datasources/heat/stack/__init__.py Normal file
View File

@ -0,0 +1,38 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from vitrage.common.constants import UpdateMethod
HEAT_STACK_DATASOURCE = 'heat.stack'
OPTS = [
cfg.StrOpt('transformer',
default='vitrage.datasources.heat.stack.transformer.'
'HeatStackTransformer',
help='Heat stack transformer class path',
required=True),
cfg.StrOpt('driver',
default='vitrage.datasources.heat.stack.driver.'
'HeatStackDriver',
help='Heat stack driver class path',
required=True),
cfg.StrOpt('update_method',
default=UpdateMethod.PUSH,
help='None: updates only via Vitrage periodic snapshots.'
'Pull: updates every [changes_interval] seconds.'
'Push: updates by getting notifications from the'
' datasource itself.',
required=True),
]

117
vitrage/datasources/heat/stack/driver.py Normal file
View File

@ -0,0 +1,117 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from vitrage import clients
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import SyncMode
from vitrage.datasources.cinder.volume import CINDER_VOLUME_DATASOURCE
from vitrage.datasources.driver_base import DriverBase
from vitrage.datasources.heat.stack import HEAT_STACK_DATASOURCE
from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE
from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE
from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE
LOG = logging.getLogger(__name__)
class HeatStackDriver(DriverBase):
_client = None
FAILED = 'FAILED'
RESOURCE_TYPE_CONVERSION = {
'OS::Nova::Server': NOVA_INSTANCE_DATASOURCE,
'OS::Cinder::Volume': CINDER_VOLUME_DATASOURCE,
'OS::Neutron::Net': NEUTRON_NETWORK_DATASOURCE,
'OS::Neutron::Port': NEUTRON_PORT_DATASOURCE
}
def __init__(self, conf):
super(HeatStackDriver, self).__init__()
self.conf = conf
self._filter_resource_types()
HeatStackDriver._client = self.client
@property
def client(self):
if not self._client:
self._client = clients.heat_client(self.conf)
return self._client
@staticmethod
def get_topic(conf):
return conf[HEAT_STACK_DATASOURCE].notification_topic
@staticmethod
def get_event_types(conf):
return ['orchestration.stack.create.end',
'orchestration.stack.delete.end',
'orchestration.stack.update.error',
'orchestration.stack.update.end',
'orchestration.stack.suspend.error',
'orchestration.stack.suspend.end',
'orchestration.stack.resume.error',
'orchestration.stack.resume.end']
@staticmethod
def enrich_event(event, event_type):
# TODO(Nofar): add call to get resources of the stack if not deleted
# change transformer that if delete we remove the stack from the graph
# and hence all the edges to it
event[DSProps.EVENT_TYPE] = event_type
event = HeatStackDriver._retrieve_stack_resources(
event, event['stack_identity'])
return HeatStackDriver.make_pickleable([event],
HEAT_STACK_DATASOURCE,
SyncMode.UPDATE)[0]
def _filter_resource_types(self):
types = self.conf.datasources.types
tmp_dict = {}
for key, value in HeatStackDriver.RESOURCE_TYPE_CONVERSION.items():
if value in types:
tmp_dict[key] = value
HeatStackDriver.RESOURCE_TYPE_CONVERSION = tmp_dict
def _make_stacks_list(self, stacks):
return [stack.__dict__ for stack in stacks
if self.FAILED not in stack.__dict__['stack_status']]
def _append_stacks_resources(self, stacks):
return [self._retrieve_stack_resources(stack, stack['id'])
for stack in stacks]
@classmethod
def _retrieve_stack_resources(cls, stack, stack_id):
resources = cls._client.resources.list(stack_id)
stack['resources'] = [resource.__dict__ for resource in resources
if resource.__dict__['resource_type'] in
HeatStackDriver.RESOURCE_TYPE_CONVERSION]
return stack
def get_all(self, sync_mode):
stacks = self.client.stacks.list(global_tenant=True)
stacks_list = self._make_stacks_list(stacks)
stacks_with_resources = self._append_stacks_resources(stacks_list)
return self.make_pickleable(stacks_with_resources,
HEAT_STACK_DATASOURCE,
sync_mode,
'manager')

158
vitrage/datasources/heat/stack/transformer.py Normal file
View File

@ -0,0 +1,158 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EdgeLabel
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import EventAction
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.cinder.volume import CINDER_VOLUME_DATASOURCE
from vitrage.datasources.heat.stack import HEAT_STACK_DATASOURCE
from vitrage.datasources.neutron.network import NEUTRON_NETWORK_DATASOURCE
from vitrage.datasources.neutron.port import NEUTRON_PORT_DATASOURCE
from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE
from vitrage.datasources.resource_transformer_base import \
ResourceTransformerBase
from vitrage.datasources import transformer_base as tbase
from vitrage.datasources.transformer_base import build_key
from vitrage.datasources.transformer_base import extract_field_value
from vitrage.datasources.transformer_base import Neighbor
import vitrage.graph.utils as graph_utils
LOG = logging.getLogger(__name__)
class HeatStackTransformer(ResourceTransformerBase):
RESOURCE_TYPE_CONVERSION = {
'OS::Nova::Server': NOVA_INSTANCE_DATASOURCE,
'OS::Cinder::Volume': CINDER_VOLUME_DATASOURCE,
'OS::Neutron::Net': NEUTRON_NETWORK_DATASOURCE,
'OS::Neutron::Port': NEUTRON_PORT_DATASOURCE
}
# Event types which need to refer them differently
UPDATE_EVENT_TYPES = {
'orchestration.stack.delete.end': EventAction.DELETE_ENTITY,
}
def __init__(self, transformers, conf):
super(HeatStackTransformer, self).__init__(transformers, conf)
def _create_snapshot_entity_vertex(self, entity_event):
stack_name = extract_field_value(entity_event, 'stack_name')
stack_id = extract_field_value(entity_event, 'id')
stack_state = extract_field_value(entity_event, 'stack_status')
timestamp = extract_field_value(entity_event, 'creation_time')
project_id = extract_field_value(entity_event, 'project_id')
return self._create_vertex(entity_event,
stack_name,
stack_id,
stack_state,
timestamp,
project_id)
def _create_update_entity_vertex(self, entity_event):
volume_name = extract_field_value(entity_event, 'stack_name')
volume_id = extract_field_value(entity_event, 'stack_identity')
volume_state = extract_field_value(entity_event, 'state')
timestamp = entity_event.get('create_at', None)
project_id = entity_event.get('tenant_id', None)
return self._create_vertex(entity_event,
volume_name,
volume_id,
volume_state,
timestamp,
project_id)
def _create_vertex(self,
entity_event,
stack_name,
stack_id,
stack_state,
update_timestamp,
project_id):
metadata = {
VProps.NAME: stack_name,
VProps.PROJECT_ID: project_id,
}
entity_key = self._create_entity_key(entity_event)
sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
return graph_utils.create_vertex(
entity_key,
entity_id=stack_id,
entity_category=EntityCategory.RESOURCE,
entity_type=HEAT_STACK_DATASOURCE,
entity_state=stack_state,
sample_timestamp=sample_timestamp,
update_timestamp=update_timestamp,
metadata=metadata)
def _create_snapshot_neighbors(self, entity_event):
return self._create_neighbors(entity_event)
def _create_update_neighbors(self, entity_event):
return self._create_neighbors(entity_event)
def _create_entity_key(self, entity_event):
is_update_event = tbase.is_update_event(entity_event)
id_field_path = 'stack_identity' if is_update_event else 'id'
volume_id = extract_field_value(entity_event, id_field_path)
key_fields = self._key_values(HEAT_STACK_DATASOURCE, volume_id)
return build_key(key_fields)
def _create_neighbors(self, entity_event):
return [self._create_neighbor(entity_event, neighbor)
for neighbor in entity_event['resources']]
def _create_neighbor(self,
entity_event,
neighbor):
datasource_type = \
self.RESOURCE_TYPE_CONVERSION[neighbor['resource_type']]
transformer = self.transformers.get(datasource_type, None)
stack_vitrage_id = self._create_entity_key(entity_event)
neighbor_id = neighbor['physical_resource_id']
sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
properties = {
VProps.ID: neighbor_id,
VProps.TYPE: datasource_type,
VProps.SAMPLE_TIMESTAMP: sample_timestamp
}
instance_vertex = transformer.create_placeholder_vertex(**properties)
relationship_edge = graph_utils.create_edge(
source_id=stack_vitrage_id,
target_id=instance_vertex.vertex_id,
relationship_type=EdgeLabel.COMPRISED)
return Neighbor(instance_vertex, relationship_edge)
def get_type(self):
return HEAT_STACK_DATASOURCE

2
vitrage/datasources/zabbix/driver.py
View File

@ -38,7 +38,7 @@ class ZabbixDriver(AlarmDriverBase):
def __init__(self, conf):
super(ZabbixDriver, self).__init__()
self.conf = conf
if ZabbixDriver.conf_map is None:
if not ZabbixDriver.conf_map:
ZabbixDriver.conf_map =\
ZabbixDriver._configuration_mapping(conf)
self._client = None