Merge "Add Trove datasource"

This commit is contained in:
Zuul 2018-12-20 23:26:54 +00:00 committed by Gerrit Code Review
commit 8124d1a419
26 changed files with 1131 additions and 6 deletions

View File

@ -184,6 +184,11 @@ function configure_vitrage {
disable_vitrage_datasource heat.stack disable_vitrage_datasource heat.stack
fi fi
# remove trove vitrage datasource if trove datasource not installed
if ! is_service_enabled trove; then
disable_vitrage_datasource trove.instance trove.cluster
fi
# remove nagios vitrage datasource if nagios datasource not installed # remove nagios vitrage datasource if nagios datasource not installed
if [[ "$VITRAGE_USE_NAGIOS" == "False" ]]; then if [[ "$VITRAGE_USE_NAGIOS" == "False" ]]; then
disable_vitrage_datasource nagios disable_vitrage_datasource nagios

View File

@ -32,7 +32,7 @@ VITRAGE_SERVICE_PORT=${VITRAGE_SERVICE_PORT:-8999}
# Toggle for deploying Vitrage with/without nagios # Toggle for deploying Vitrage with/without nagios
VITRAGE_USE_NAGIOS=$(trueorfalse False VITRAGE_USE_NAGIOS) VITRAGE_USE_NAGIOS=$(trueorfalse False VITRAGE_USE_NAGIOS)
VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus} VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus,trove.instance,trove.cluster}
# for now dont use pip install for the client # for now dont use pip install for the client
LIBS_FROM_GIT=python-vitrageclient LIBS_FROM_GIT=python-vitrageclient

View File

@ -0,0 +1,31 @@
category: RESOURCE
values:
- aggregated values:
priority: 50
original values:
- name: DELETED
operational_value: DELETED
- aggregated values:
priority: 40
original values:
- name: ERROR
operational_value: ERROR
- aggregated values:
priority: 30
original values:
- name: GROWING
operational_value: TRANSIENT
- name: SHRINKING
operational_value: TRANSIENT
- name: UPGRADING
operational_value: TRANSIENT
- aggregated values:
priority: 20
original values:
- name: SUBOPTIMAL
operational_value: SUBOPTIMAL
- aggregated values:
priority: 10
original values:
- name: NONE
operational_value: OK

View File

@ -0,0 +1,47 @@
category: RESOURCE
values:
- aggregated values:
priority: 70
original values:
- name: DELETED
operational_value: DELETED
- aggregated values:
priority: 60
original values:
- name: FAILED
operational_value: ERROR
- name: ERROR
operational_value: ERROR
- aggregated values:
priority: 50
original values:
- name: SHUTDOWN
operational_value: SUBOPTIMAL
- name: BLOCKED
operational_value: SUBOPTIMAL
- aggregated values:
priority: 40
original values:
- name: RESIZING
operational_value: TRANSIENT
- name: UPGRADING
operational_value: TRANSIENT
- name: REBOOTING
operational_value: TRANSIENT
- aggregated values:
priority: 30
original values:
- name: NEW
operational_value: TRANSIENT
- name: BUILDING
operational_value: TRANSIENT
- aggregated values:
priority: 20
original values:
- name: SUBOPTIMAL
operational_value: SUBOPTIMAL
- aggregated values:
priority: 10
original values:
- name: ACTIVE
operational_value: OK

View File

@ -103,6 +103,7 @@ python-novaclient==10.1.0
python-openstackclient==3.12.0 python-openstackclient==3.12.0
python-subunit==1.2.0 python-subunit==1.2.0
python-swiftclient==3.5.0 python-swiftclient==3.5.0
python-troveclient==2.2.0
pytz==2013.6 pytz==2013.6
PyYAML==3.12 PyYAML==3.12
pyzabbix==0.7.4 pyzabbix==0.7.4

View File

@ -0,0 +1,14 @@
---
features:
- A new ``Trove Datasource`` has been introduced to include Trove entities
(database instances and clusters) in Vitrage Entity Graph. Trove is Database
as a Service solution offering database lifecycle management (automated
provisioning, configuration, backups, clustering etc.). Adding the
datasource to Vitrage enables detecting problems at lower levels of
infrastructure that may affect functioning of running databases, and react
in response to identified issues e.g. scale the database up/out or
live-migrate virtual machines from failed compute. This change is the
first stage of integration with Trove. At this point, Trove entities are
extracted using PULL approach, based on periodical snapshot-query to Trove
API for the current list of Trove entities. In the future, PUSH approach
based on Trove notifications will be implemented.

View File

@ -15,6 +15,7 @@ python-novaclient>=10.1.0 # Apache-2.0
python-heatclient>=1.14.0 # Apache-2.0 python-heatclient>=1.14.0 # Apache-2.0
python-mistralclient>=3.3.0 # Apache-2.0 python-mistralclient>=3.3.0 # Apache-2.0
python-openstackclient>=3.12.0 # Apache-2.0 python-openstackclient>=3.12.0 # Apache-2.0
python-troveclient>=2.2.0 # Apache-2.0
gnocchiclient>=3.3.1 # Apache-2.0 gnocchiclient>=3.3.1 # Apache-2.0
pyzabbix>=0.7.4 # LGPL pyzabbix>=0.7.4 # LGPL
networkx>=2.0 # BSD networkx>=2.0 # BSD

View File

View File

@ -0,0 +1,44 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import UpdateMethod
TROVE_CLUSTER_DATASOURCE = 'trove.cluster'
OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.trove.cluster.transformer.'
'TroveClusterTransformer',
help='Trove transformer class path.',
required=True),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.trove.cluster.driver.'
'TroveClusterDriver',
help='Trove driver class path.',
required=True),
cfg.StrOpt(DSOpts.UPDATE_METHOD,
default=UpdateMethod.PULL,
help='None: updates only via Vitrage periodic snapshots.'
'Pull: updates periodically.'
'Push: updates by getting notifications from the'
' datasource itself.',
required=True),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=10,
help='Interval in seconds between checking changes in Trove'
'cluster datasource.')]

View File

@ -0,0 +1,51 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from vitrage.datasources.transformer_base import extract_field_value
from vitrage.datasources.trove.cluster import TROVE_CLUSTER_DATASOURCE
from vitrage.datasources.trove.properties import \
TroveClusterProperties as TProps
from vitrage.datasources.trove.trove_driver_base import TroveDriverBase
LOG = log.getLogger(__name__)
class TroveClusterDriver(TroveDriverBase):
def __init__(self, conf):
super(TroveClusterDriver, self).__init__(conf)
self._cached_entities = []
def _get_vitrage_type(self):
return TROVE_CLUSTER_DATASOURCE
def _get_all_entities(self):
# TODO(bzurkowski): Add all_tenants option to Trove client
return self.extract_entities(self.client.clusters.list())
def _find_entity(self, search_entity, entities):
for entity in entities:
if entity[TProps.ID] == search_entity[TProps.ID]:
return entity
def _equal_entities(self, old_entity, new_entity):
old_state = extract_field_value(old_entity, *TProps.STATE)
new_state = extract_field_value(old_entity, *TProps.STATE)
return old_entity[TProps.ID] == new_entity[TProps.ID] and \
old_entity[TProps.NAME] == new_entity[TProps.NAME] and \
old_state == new_state

View File

@ -0,0 +1,84 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EdgeLabel
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.resource_transformer_base import \
ResourceTransformerBase
from vitrage.datasources import transformer_base as tbase
from vitrage.datasources.transformer_base import extract_field_value
from vitrage.datasources.trove.cluster import TROVE_CLUSTER_DATASOURCE
from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE
from vitrage.datasources.trove.properties import \
TroveClusterProperties as TProps
import vitrage.graph.utils as graph_utils
class TroveClusterTransformer(ResourceTransformerBase):
def _create_snapshot_entity_vertex(self, entity_event):
return self._create_vertex(entity_event)
def _create_update_entity_vertex(self, entity_event):
return self._create_vertex(entity_event)
def _create_vertex(self, entity_event):
# TODO(bzurkowski): Add project ID
entity_id = entity_event[TProps.ID]
name = entity_event[TProps.NAME]
state = extract_field_value(entity_event, *TProps.STATE)
update_timestamp = entity_event[TProps.UPDATE_TIMESTAMP]
sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
metadata = {
VProps.NAME: name
}
return graph_utils.create_vertex(
self._create_entity_key(entity_event),
vitrage_category=EntityCategory.RESOURCE,
vitrage_type=TROVE_CLUSTER_DATASOURCE,
vitrage_sample_timestamp=sample_timestamp,
entity_id=entity_id,
update_timestamp=update_timestamp,
entity_state=state,
metadata=metadata)
def _create_snapshot_neighbors(self, entity_event):
return self._create_entity_neighbours(entity_event)
def _create_update_neighbors(self, entity_event):
return self._create_entity_neighbours(entity_event)
def _create_entity_neighbours(self, entity_event):
neighbours = []
for instance in entity_event[TProps.INSTANCES]:
instance_neighbour = self._create_neighbor(
entity_event,
instance[TProps.ID],
TROVE_INSTANCE_DATASOURCE,
EdgeLabel.CONTAINS,
is_entity_source=True)
neighbours.append(instance_neighbour)
return neighbours
def _create_entity_key(self, entity_event):
entity_id = entity_event[TProps.ID]
key_fields = self._key_values(TROVE_CLUSTER_DATASOURCE, entity_id)
return tbase.build_key(key_fields)
@staticmethod
def get_vitrage_type():
return TROVE_CLUSTER_DATASOURCE

View File

@ -0,0 +1,44 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import UpdateMethod
TROVE_INSTANCE_DATASOURCE = 'trove.instance'
OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.trove.instance.transformer.'
'TroveInstanceTransformer',
help='Trove transformer class path.',
required=True),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.trove.instance.driver.'
'TroveInstanceDriver',
help='Trove driver class path.',
required=True),
cfg.StrOpt(DSOpts.UPDATE_METHOD,
default=UpdateMethod.PULL,
help='None: updates only via Vitrage periodic snapshots.'
'Pull: updates periodically.'
'Push: updates by getting notifications from the'
' datasource itself.',
required=True),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=10,
help='Interval in seconds between checking changes in Trove'
'instance datasource.')]

View File

@ -0,0 +1,49 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE
from vitrage.datasources.trove.properties import \
TroveInstanceProperties as TProps
from vitrage.datasources.trove.trove_driver_base import TroveDriverBase
LOG = log.getLogger(__name__)
class TroveInstanceDriver(TroveDriverBase):
def __init__(self, conf):
super(TroveInstanceDriver, self).__init__(conf)
self._cached_entities = []
def _get_vitrage_type(self):
return TROVE_INSTANCE_DATASOURCE
def _get_all_entities(self):
# TODO(bzurkowski): Add all_tenants option to Trove client
return self.extract_entities(
self.client.instances.list(include_clustered=True, detailed=True))
def _find_entity(self, search_entity, entities):
for entity in entities:
if entity[TProps.ID] == search_entity[TProps.ID]:
return entity
def _equal_entities(self, old_entity, new_entity):
return old_entity[TProps.ID] == new_entity[TProps.ID] and \
old_entity[TProps.NAME] == new_entity[TProps.NAME] and \
old_entity[TProps.STATE] == new_entity[TProps.STATE]

View File

@ -0,0 +1,81 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EdgeLabel
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE
from vitrage.datasources.resource_transformer_base import \
ResourceTransformerBase
from vitrage.datasources import transformer_base as tbase
from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE
from vitrage.datasources.trove.properties import \
TroveInstanceProperties as TProps
import vitrage.graph.utils as graph_utils
class TroveInstanceTransformer(ResourceTransformerBase):
def _create_snapshot_entity_vertex(self, entity_event):
return self._create_vertex(entity_event)
def _create_update_entity_vertex(self, entity_event):
return self._create_vertex(entity_event)
def _create_vertex(self, entity_event):
entity_id = entity_event[TProps.ID]
name = entity_event[TProps.NAME]
state = entity_event[TProps.STATE]
project_id = entity_event[TProps.PROJECT_ID]
update_timestamp = entity_event[TProps.UPDATE_TIMESTAMP]
sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
metadata = {
VProps.NAME: name,
VProps.PROJECT_ID: project_id
}
return graph_utils.create_vertex(
self._create_entity_key(entity_event),
vitrage_category=EntityCategory.RESOURCE,
vitrage_type=TROVE_INSTANCE_DATASOURCE,
vitrage_sample_timestamp=sample_timestamp,
entity_id=entity_id,
update_timestamp=update_timestamp,
entity_state=state,
metadata=metadata)
def _create_snapshot_neighbors(self, entity_event):
return self._create_entity_neighbours(entity_event)
def _create_update_neighbors(self, entity_event):
return self._create_entity_neighbours(entity_event)
def _create_entity_neighbours(self, entity_event):
server_neighbour = self._create_neighbor(
entity_event,
entity_event[TProps.SERVER_ID],
NOVA_INSTANCE_DATASOURCE,
EdgeLabel.CONTAINS,
is_entity_source=True)
return [server_neighbour]
def _create_entity_key(self, entity_event):
entity_id = entity_event[TProps.ID]
key_fields = self._key_values(TROVE_INSTANCE_DATASOURCE, entity_id)
return tbase.build_key(key_fields)
@staticmethod
def get_vitrage_type():
return TROVE_INSTANCE_DATASOURCE

View File

@ -0,0 +1,34 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class TroveBaseProperties(object):
ID = 'id'
NAME = 'name'
UPDATE_TIMESTAMP = 'updated'
PROJECT_ID = 'tenant_id'
class TroveInstanceProperties(TroveBaseProperties):
STATE = 'status'
SERVER_ID = 'server_id'
class TroveClusterProperties(TroveBaseProperties):
STATE = ('task', 'name')
INSTANCES = 'instances'

View File

@ -0,0 +1,104 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import GraphAction
from vitrage.datasources.driver_base import DriverBase
from vitrage import os_clients
class TroveDriverBase(DriverBase):
def __init__(self, conf):
super(TroveDriverBase, self).__init__()
self.conf = conf
self.__client = None
self.__cached_entities = []
@property
def client(self):
if not self.__client:
self.__client = os_clients.trove_client(self.conf)
return self.__client
def get_all(self, datasource_action):
return self.make_pickleable(self._get_and_cache_all_entities(),
self._get_vitrage_type(),
datasource_action,
*self.properties_to_filter_out())
def get_changes(self, datasource_action):
return self.make_pickleable(self._get_changed_entities(),
self._get_vitrage_type(),
datasource_action,
*self.properties_to_filter_out())
def _get_and_cache_all_entities(self):
self.__cached_entities = self._get_all_entities()
return self.__cached_entities
def _get_changed_entities(self):
actual_entities = self._get_all_entities()
changed_entities = []
for actual_entity in actual_entities:
cached_entity = self._find_entity(actual_entity,
self.__cached_entities)
if cached_entity:
# Add modified entities
if not self._equal_entities(actual_entity, cached_entity):
changed_entities.append(actual_entity)
else:
# Add new entities
changed_entities.append(actual_entity)
# Delete removed entities
for cached_entity in self.__cached_entities:
if not self._find_entity(cached_entity, actual_entities):
cached_entity[DSProps.EVENT_TYPE] = GraphAction.DELETE_ENTITY
changed_entities.append(cached_entity)
self.__cached_entities = actual_entities
return changed_entities
@abc.abstractmethod
def _get_vitrage_type(self):
pass
@abc.abstractmethod
def _get_all_entities(self):
pass
@abc.abstractmethod
def _find_entity(self, search_entity, entities):
pass
@abc.abstractmethod
def _equal_entities(self, old_entity, new_entity):
pass
@staticmethod
def properties_to_filter_out():
return ['manager', '_info']
@staticmethod
def should_delete_outdated_entities():
return True
@staticmethod
def extract_entities(entities):
return [entity.to_dict() for entity in entities]

View File

@ -29,6 +29,7 @@ OPTS = [
cfg.StrOpt('heat_version', default='1', help='Heat version'), cfg.StrOpt('heat_version', default='1', help='Heat version'),
cfg.StrOpt('mistral_version', default='2', help='Mistral version'), cfg.StrOpt('mistral_version', default='2', help='Mistral version'),
cfg.StrOpt('gnocchi_version', default='1', help='Gnocchi version'), cfg.StrOpt('gnocchi_version', default='1', help='Gnocchi version'),
cfg.StrOpt('trove_version', default='1', help='Trove version'),
cfg.BoolOpt('use_nova_versioned_notifications', cfg.BoolOpt('use_nova_versioned_notifications',
default=True, default=True,
help='Indicates whether to use Nova versioned notifications.' help='Indicates whether to use Nova versioned notifications.'
@ -47,7 +48,8 @@ _client_modules = {
'neutron': 'neutronclient.v2_0.client', 'neutron': 'neutronclient.v2_0.client',
'heat': 'heatclient.client', 'heat': 'heatclient.client',
'mistral': 'mistralclient.api.v2.client', 'mistral': 'mistralclient.api.v2.client',
'gnocchi': 'gnocchiclient.v1.client' 'gnocchi': 'gnocchiclient.v1.client',
'trove': 'troveclient.v1.client'
} }
@ -110,6 +112,20 @@ def nova_client(conf):
LOG.exception('Create Nova client - Got Exception.') LOG.exception('Create Nova client - Got Exception.')
def trove_client(conf):
"""Get an instance of trove client"""
try:
tr_client = driver_module('trove')
client = tr_client.Client(
version=conf.trove_version,
session=keystone_client.get_session(conf),
)
LOG.info('Trove client created')
return client
except Exception:
LOG.exception('Create Trove client - Got Exception.')
def cinder_client(conf): def cinder_client(conf):
"""Get an instance of cinder client""" """Get an instance of cinder client"""
try: try:

View File

@ -0,0 +1,92 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from testtools import matchers
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.nagios import NAGIOS_DATASOURCE
from vitrage.datasources import NOVA_HOST_DATASOURCE
from vitrage.datasources import NOVA_INSTANCE_DATASOURCE
from vitrage.datasources import NOVA_ZONE_DATASOURCE
from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE
from vitrage.tests.functional.datasources.base import TestDataSourcesBase
from vitrage.tests.mocks import mock_driver
class TestTroveInstance(TestDataSourcesBase):
DATASOURCES_OPTS = [
cfg.ListOpt('types',
default=[NAGIOS_DATASOURCE,
NOVA_HOST_DATASOURCE,
NOVA_INSTANCE_DATASOURCE,
NOVA_ZONE_DATASOURCE,
TROVE_INSTANCE_DATASOURCE],
help='Names of supported driver data sources'),
cfg.ListOpt('path',
default=['vitrage.datasources'],
help='Base path for data sources')
]
# noinspection PyPep8Naming
@classmethod
def setUpClass(cls):
super(TestTroveInstance, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph')
cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources')
cls.load_datasources(cls.conf)
def test_trove_instance_validity(self):
# Setup
processor = self._create_processor_with_graph(self.conf)
self.assertThat(processor.entity_graph,
matchers.HasLength(
self._num_total_expected_vertices())
)
spec_list = mock_driver.simple_trove_instance_generators(
inst_num=1,
snapshot_events=1)
static_events = mock_driver.generate_random_events_list(spec_list)
trove_instance_event = static_events[0]
trove_instance_event['server_id'] = \
self._find_entity_id_by_type(processor.entity_graph,
NOVA_INSTANCE_DATASOURCE)
# Action
processor.process_event(trove_instance_event)
# Test assertions
self.assertThat(processor.entity_graph,
matchers.HasLength(
self._num_total_expected_vertices() + 1)
)
trove_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter={
VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE,
VProps.VITRAGE_TYPE: TROVE_INSTANCE_DATASOURCE
})
self.assertThat(trove_vertices, matchers.HasLength(1))
trove_neighbors = processor.entity_graph.neighbors(
trove_vertices[0].vertex_id)
self.assertThat(trove_neighbors, matchers.HasLength(1))
self.assertEqual(NOVA_INSTANCE_DATASOURCE,
trove_neighbors[0][VProps.VITRAGE_TYPE])

View File

@ -203,6 +203,60 @@ def simple_zone_generators(zone_num, host_num, snapshot_events=0,
return tg.get_trace_generators(test_entity_spec_list) return tg.get_trace_generators(test_entity_spec_list)
def simple_trove_instance_generators(inst_num, snapshot_events=0,
snap_vals=None):
"""A function for returning Trove instance generators.
Returns generators for a given number of Trove instances.
:param inst_num: number of instances
:return: generators for inst_num instances as specified
"""
mapping = [('tr-instance-{0}'.format(idx), 'vm-{0}'.format(idx))
for idx in range(inst_num)]
test_entity_spec_list = [
{tg.DYNAMIC_INFO_FKEY: tg.DRIVER_TROVE_INSTANCE_SNAPSHOT_D,
tg.STATIC_INFO_FKEY: None,
tg.MAPPING_KEY: mapping,
tg.EXTERNAL_INFO_KEY: snap_vals,
tg.NAME_KEY: 'Database instance snapshot generator',
tg.NUM_EVENTS: snapshot_events
}
]
return tg.get_trace_generators(test_entity_spec_list)
def simple_trove_cluster_generators(clust_num, inst_num, snapshot_events=0,
snap_vals=None):
"""A function for returning Trove cluster generators.
Returns generators for a given number of Trove clusters.
:param clust_num: number of clusters
:param inst_num: number of instances
:return: generators for clust_num clusters as specified
"""
mapping = [('tr-cluster-{0}'.format(idx % clust_num),
'tr-inst-{0}'.format(idx))
for idx in range(inst_num)]
test_entity_spec_list = [
{tg.DYNAMIC_INFO_FKEY: tg.DRIVER_TROVE_CLUSTER_SNAPSHOT_D,
tg.STATIC_INFO_FKEY: None,
tg.MAPPING_KEY: mapping,
tg.EXTERNAL_INFO_KEY: snap_vals,
tg.NAME_KEY: 'Database cluster snapshot generator',
tg.NUM_EVENTS: snapshot_events
}
]
return tg.get_trace_generators(test_entity_spec_list)
def simple_volume_generators(volume_num, instance_num, def simple_volume_generators(volume_num, instance_num,
snapshot_events=0, update_events=0, snapshot_events=0, update_events=0,
snap_vals=None, update_vals=None): snap_vals=None, update_vals=None):

View File

@ -68,7 +68,10 @@ DRIVER_STACK_SNAPSHOT_D = 'driver_stack_snapshot_dynamic.json'
DRIVER_CONSISTENCY_UPDATE_D = 'driver_consistency_update_dynamic.json' DRIVER_CONSISTENCY_UPDATE_D = 'driver_consistency_update_dynamic.json'
DRIVER_ZONE_SNAPSHOT_D = 'driver_zone_snapshot_dynamic.json' DRIVER_ZONE_SNAPSHOT_D = 'driver_zone_snapshot_dynamic.json'
DRIVER_KUBE_SNAPSHOT_D = 'driver_kubernetes_snapshot_dynamic.json' DRIVER_KUBE_SNAPSHOT_D = 'driver_kubernetes_snapshot_dynamic.json'
DRIVER_TROVE_INSTANCE_SNAPSHOT_D = \
'driver_trove_instance_snapshot_dynamic.json'
DRIVER_TROVE_CLUSTER_SNAPSHOT_D = \
'driver_trove_cluster_snapshot_dynamic.json'
# Mock transformer Specs (i.e., what the transformer outputs) # Mock transformer Specs (i.e., what the transformer outputs)
MOCK_TRANSFORMER_PATH = '%s/mock_configurations/transformer' % \ MOCK_TRANSFORMER_PATH = '%s/mock_configurations/transformer' % \
@ -79,7 +82,6 @@ TRANS_DOCTOR_UPDATE_D = 'transformer_doctor_update_dynamic.json'
TRANS_COLLECTD_UPDATE_D = 'transformer_collectd_update_dynamic.json' TRANS_COLLECTD_UPDATE_D = 'transformer_collectd_update_dynamic.json'
TRANS_PROMETHEUS_UPDATE_D = 'transformer_prometheus_update_dynamic.json' TRANS_PROMETHEUS_UPDATE_D = 'transformer_prometheus_update_dynamic.json'
TRANS_INST_SNAPSHOT_D = 'transformer_inst_snapshot_dynamic.json' TRANS_INST_SNAPSHOT_D = 'transformer_inst_snapshot_dynamic.json'
TRANS_INST_SNAPSHOT_S = 'transformer_inst_snapshot_static.json'
TRANS_HOST_SNAPSHOT_D = 'transformer_host_snapshot_dynamic.json' TRANS_HOST_SNAPSHOT_D = 'transformer_host_snapshot_dynamic.json'
TRANS_HOST_SNAPSHOT_S = 'transformer_host_snapshot_static.json' TRANS_HOST_SNAPSHOT_S = 'transformer_host_snapshot_static.json'
TRANS_ZONE_SNAPSHOT_D = 'transformer_zone_snapshot_dynamic.json' TRANS_ZONE_SNAPSHOT_D = 'transformer_zone_snapshot_dynamic.json'
@ -139,7 +141,10 @@ class EventTraceGenerator(object):
DRIVER_CONSISTENCY_UPDATE_D: DRIVER_CONSISTENCY_UPDATE_D:
_get_consistency_update_driver_values, _get_consistency_update_driver_values,
DRIVER_PROMETHEUS_UPDATE_D: _get_simple_update_driver_values, DRIVER_PROMETHEUS_UPDATE_D: _get_simple_update_driver_values,
DRIVER_TROVE_INSTANCE_SNAPSHOT_D:
_get_trove_instance_snapshot_driver_values,
DRIVER_TROVE_CLUSTER_SNAPSHOT_D:
_get_trove_cluster_snapshot_driver_values,
TRANS_AODH_SNAPSHOT_D: _get_trans_aodh_alarm_snapshot_values, TRANS_AODH_SNAPSHOT_D: _get_trans_aodh_alarm_snapshot_values,
TRANS_AODH_UPDATE_D: _get_trans_aodh_alarm_snapshot_values, TRANS_AODH_UPDATE_D: _get_trans_aodh_alarm_snapshot_values,
TRANS_DOCTOR_UPDATE_D: _get_simple_trans_alarm_update_values, TRANS_DOCTOR_UPDATE_D: _get_simple_trans_alarm_update_values,
@ -664,6 +669,40 @@ def _get_zabbix_alarm_driver_values(spec):
return static_values return static_values
def _get_trove_instance_snapshot_driver_values(spec):
inst_srv_mapping = spec[MAPPING_KEY]
static_info = None
if spec[STATIC_INFO_FKEY] is not None:
static_info = utils.load_specs(spec[STATIC_INFO_FKEY])
static_values = []
for inst_name, srv_name in inst_srv_mapping:
mapping = {'id': inst_name,
'name': inst_name,
'server_id': srv_name}
static_values.append(combine_data(
static_info, mapping, spec.get(EXTERNAL_INFO_KEY, None)))
return static_values
def _get_trove_cluster_snapshot_driver_values(spec):
clust_inst_mapping = spec[MAPPING_KEY]
static_info = None
if spec[STATIC_INFO_FKEY] is not None:
static_info = utils.load_specs(spec[STATIC_INFO_FKEY])
static_values = []
for clust_name, inst_name in clust_inst_mapping:
mapping = {'id': clust_name,
'name': clust_name,
'instances': [
{'id': inst_name, 'name': inst_name}
]}
static_values.append(combine_data(
static_info, mapping, spec.get(EXTERNAL_INFO_KEY, None)))
return static_values
def _get_trans_host_snapshot_values(spec): def _get_trans_host_snapshot_values(spec):
"""Generates the static driver values for each host. """Generates the static driver values for each host.

View File

@ -0,0 +1,19 @@
{
"id": "tr-clust-0",
"name": "tr-clust-0",
"task": {
"id": 1,
"name": "NONE"
},
"instances": [
{
"id": "tr-inst-0",
"name": "tr-inst-0"
}
],
"created": "2018-10-01T12:45:30Z",
"updated": "2018-10-01T12:46:00Z",
"vitrage_entity_type": "trove.cluster",
"vitrage_datasource_action": "snapshot",
"vitrage_sample_date": "2018-10-01 12:50:00.000000+00:00"
}

View File

@ -0,0 +1,13 @@
{
"id": "tr-inst-0",
"name": "tr-inst-0",
"status": "ACTIVE",
"server_id": "vm-0",
"volume_id": "volume-0",
"tenant_id": "tenant-0",
"created": "2018-10-01T12:45:30Z",
"updated": "2018-10-01T12:46:00Z",
"vitrage_entity_type": "trove.instance",
"vitrage_datasource_action": "snapshot",
"vitrage_sample_date": "2018-10-01 12:50:00.000000+00:00"
}

View File

@ -0,0 +1,151 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_config import cfg
from oslo_log import log as logging
from testtools import matchers
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EdgeLabel
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import UpdateMethod
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources import transformer_base as tb
from vitrage.datasources.transformer_base import TransformerBase
from vitrage.datasources.trove.cluster.transformer import \
TroveClusterTransformer
from vitrage.datasources.trove.cluster import TROVE_CLUSTER_DATASOURCE
from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE
from vitrage.tests import base
from vitrage.tests.mocks import mock_driver as mock_sync
LOG = logging.getLogger(__name__)
class TroveClusterTransformerTest(base.BaseTest):
OPTS = [
cfg.StrOpt(DSOpts.UPDATE_METHOD, default=UpdateMethod.PULL),
]
@classmethod
def setUpClass(cls):
super(TroveClusterTransformerTest, cls).setUpClass()
cls.transformers = {}
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group=TROVE_CLUSTER_DATASOURCE)
cls.transformers[TROVE_CLUSTER_DATASOURCE] = \
TroveClusterTransformer(cls.transformers, cls.conf)
def test_create_placeholder_vertex(self):
# Tests setup
cluster_id = 'tr-cluster-0'
timestamp = datetime.datetime.utcnow()
properties = {
VProps.ID: cluster_id,
VProps.VITRAGE_TYPE: TROVE_CLUSTER_DATASOURCE,
VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE,
VProps.VITRAGE_SAMPLE_TIMESTAMP: timestamp
}
transformer = self.transformers[TROVE_CLUSTER_DATASOURCE]
# Test action
placeholder = transformer.create_neighbor_placeholder_vertex(
**properties)
# Test assertions
expected_key = tb.build_key(
transformer._key_values(TROVE_CLUSTER_DATASOURCE, cluster_id))
expected_uuid = TransformerBase.uuid_from_deprecated_vitrage_id(
expected_key)
self.assertEqual(expected_uuid, placeholder.vertex_id)
self.assertEqual(timestamp,
placeholder.get(VProps.VITRAGE_SAMPLE_TIMESTAMP))
self.assertEqual(TROVE_CLUSTER_DATASOURCE,
placeholder.get(VProps.VITRAGE_TYPE))
self.assertEqual(cluster_id, placeholder.get(VProps.ID))
self.assertEqual(EntityCategory.RESOURCE,
placeholder.get(VProps.VITRAGE_CATEGORY))
self.assertTrue(placeholder.get(VProps.VITRAGE_IS_PLACEHOLDER))
def test_snapshot_event_transform(self):
# Test setup
spec_list = mock_sync.simple_trove_cluster_generators(
clust_num=1, inst_num=1, snapshot_events=10)
events = mock_sync.generate_random_events_list(spec_list)
for event in events:
# Test action
transformer = self.transformers[TROVE_CLUSTER_DATASOURCE]
wrapper = transformer.transform(event)
# Test assertions
vertex = wrapper.vertex
self._validate_vertex_props(vertex, event)
neighbours = wrapper.neighbors
self.assertThat(neighbours, matchers.HasLength(1))
self._validate_server_neighbour(neighbours[0], vertex.vertex_id,
event)
def _validate_vertex_props(self, vertex, event):
self.assertEqual(event['id'], vertex[VProps.ID])
self.assertEqual(event['name'], vertex[VProps.NAME])
self.assertEqual(event['task']['name'], vertex[VProps.STATE])
self.assertEqual(event[DSProps.SAMPLE_DATE],
vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP])
self.assertEqual(EntityCategory.RESOURCE,
vertex[VProps.VITRAGE_CATEGORY])
self.assertEqual(TROVE_CLUSTER_DATASOURCE,
vertex[VProps.VITRAGE_TYPE])
self.assertFalse(vertex[VProps.VITRAGE_IS_PLACEHOLDER])
self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED])
def _validate_server_neighbour(self, neighbour, cluster_id, event):
vertex, edge = neighbour.vertex, neighbour.edge
# Validate neighbor vertex
self.assertEqual(EntityCategory.RESOURCE,
vertex[VProps.VITRAGE_CATEGORY])
self.assertEqual(TROVE_INSTANCE_DATASOURCE,
vertex[VProps.VITRAGE_TYPE])
instance_id = event['instances'][0]['id']
self.assertEqual(instance_id, vertex[VProps.ID])
self.assertTrue(vertex[VProps.VITRAGE_IS_PLACEHOLDER])
self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED])
# Validate neighbor edge
self.assertEqual(edge.target_id, vertex.vertex_id)
self.assertEqual(edge.source_id, cluster_id)
self.assertEqual(edge.label, EdgeLabel.CONTAINS)

View File

@ -0,0 +1,151 @@
# Copyright 2018 Samsung Electronics
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_config import cfg
from oslo_log import log as logging
from testtools import matchers
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EdgeLabel
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import UpdateMethod
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE
from vitrage.datasources import transformer_base as tb
from vitrage.datasources.transformer_base import TransformerBase
from vitrage.datasources.trove.instance.transformer import \
TroveInstanceTransformer
from vitrage.datasources.trove.instance import TROVE_INSTANCE_DATASOURCE
from vitrage.tests import base
from vitrage.tests.mocks import mock_driver as mock_sync
LOG = logging.getLogger(__name__)
class TroveInstanceTransformerTest(base.BaseTest):
OPTS = [
cfg.StrOpt(DSOpts.UPDATE_METHOD, default=UpdateMethod.PULL),
]
@classmethod
def setUpClass(cls):
super(TroveInstanceTransformerTest, cls).setUpClass()
cls.transformers = {}
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group=TROVE_INSTANCE_DATASOURCE)
cls.transformers[TROVE_INSTANCE_DATASOURCE] = \
TroveInstanceTransformer(cls.transformers, cls.conf)
def test_create_placeholder_vertex(self):
# Tests setup
instance_id = 'tr-instance-0'
timestamp = datetime.datetime.utcnow()
properties = {
VProps.ID: instance_id,
VProps.VITRAGE_TYPE: TROVE_INSTANCE_DATASOURCE,
VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE,
VProps.VITRAGE_SAMPLE_TIMESTAMP: timestamp
}
transformer = self.transformers[TROVE_INSTANCE_DATASOURCE]
# Test action
placeholder = transformer.create_neighbor_placeholder_vertex(
**properties)
# Test assertions
expected_key = tb.build_key(
transformer._key_values(TROVE_INSTANCE_DATASOURCE, instance_id))
expected_uuid = TransformerBase.uuid_from_deprecated_vitrage_id(
expected_key)
self.assertEqual(expected_uuid, placeholder.vertex_id)
self.assertEqual(timestamp,
placeholder.get(VProps.VITRAGE_SAMPLE_TIMESTAMP))
self.assertEqual(TROVE_INSTANCE_DATASOURCE,
placeholder.get(VProps.VITRAGE_TYPE))
self.assertEqual(instance_id, placeholder.get(VProps.ID))
self.assertEqual(EntityCategory.RESOURCE,
placeholder.get(VProps.VITRAGE_CATEGORY))
self.assertTrue(placeholder.get(VProps.VITRAGE_IS_PLACEHOLDER))
def test_snapshot_event_transform(self):
# Test setup
spec_list = mock_sync.simple_trove_instance_generators(
inst_num=1, snapshot_events=10)
events = mock_sync.generate_random_events_list(spec_list)
for event in events:
# Test action
transformer = self.transformers[TROVE_INSTANCE_DATASOURCE]
wrapper = transformer.transform(event)
# Test assertions
vertex = wrapper.vertex
self._validate_vertex_props(vertex, event)
neighbours = wrapper.neighbors
self.assertThat(neighbours, matchers.HasLength(1))
self._validate_server_neighbour(neighbours[0], vertex.vertex_id,
event)
def _validate_vertex_props(self, vertex, event):
self.assertEqual(event['id'], vertex[VProps.ID])
self.assertEqual(event['name'], vertex[VProps.NAME])
self.assertEqual(event['status'], vertex[VProps.STATE])
self.assertEqual(event['tenant_id'], vertex[VProps.PROJECT_ID])
self.assertEqual(event[DSProps.SAMPLE_DATE],
vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP])
self.assertEqual(EntityCategory.RESOURCE,
vertex[VProps.VITRAGE_CATEGORY])
self.assertEqual(TROVE_INSTANCE_DATASOURCE,
vertex[VProps.VITRAGE_TYPE])
self.assertFalse(vertex[VProps.VITRAGE_IS_PLACEHOLDER])
self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED])
def _validate_server_neighbour(self, neighbour, instance_id, event):
vertex, edge = neighbour.vertex, neighbour.edge
# Validate neighbor vertex
self.assertEqual(EntityCategory.RESOURCE,
vertex[VProps.VITRAGE_CATEGORY])
self.assertEqual(NOVA_INSTANCE_DATASOURCE, vertex[VProps.VITRAGE_TYPE])
self.assertEqual(event['server_id'], vertex[VProps.ID])
self.assertTrue(vertex[VProps.VITRAGE_IS_PLACEHOLDER])
self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED])
# Validate neighbor edge
self.assertEqual(edge.target_id, vertex.vertex_id)
self.assertEqual(edge.source_id, instance_id)
self.assertEqual(edge.label, EdgeLabel.CONTAINS)