Merge "Implement static datasource transformer"

This commit is contained in:
Jenkins 2017-01-12 09:01:44 +00:00 committed by Gerrit Code Review
commit cbdddc4e4c
12 changed files with 390 additions and 52 deletions

View File

@ -13,6 +13,7 @@
# under the License.
from oslo_config import cfg
from vitrage.common.constants import TopologyFields
from vitrage.common.constants import UpdateMethod
STATIC_DATASOURCE = 'static'
@ -43,5 +44,8 @@ OPTS = [
# configuration files will NOT be converted automatically. But user will
# receive deprecation warnings.
cfg.StrOpt('directory', default='/etc/vitrage/static_datasources',
help='static data sources configuration directory')
]
help='static data sources configuration directory')]
class StaticFields(TopologyFields):
CONFIG_ID = 'config_id'

View File

@ -13,30 +13,31 @@
# under the License.
from itertools import chain
from six import iteritems
from six.moves import reduce
from oslo_log import log
from vitrage.common.constants import TopologyFields
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.driver_base import DriverBase
from vitrage.datasources.static import STATIC_DATASOURCE
from vitrage.datasources.static import StaticFields
from vitrage.utils import file as file_utils
LOG = log.getLogger(__name__)
class StaticDriver(DriverBase):
CONFIG_ID = 'config_id'
# base fields are required for all entities, others are treated as metadata
BASE_FIELDS = {StaticFields.CONFIG_ID, StaticFields.TYPE, VProps.ID}
def __init__(self, conf):
super(StaticDriver, self).__init__()
self.cfg = conf
self.cache = {}
self.legacy_driver = {}
@staticmethod
def is_valid_config(config):
def _is_valid_config(config):
"""check for validity of configuration"""
# TODO(yujunz) check with yaml schema or reuse template validation
return TopologyFields.DEFINITIONS in config
@ -71,7 +72,7 @@ class StaticDriver(DriverBase):
def _get_entities_from_file(cls, path):
config = file_utils.load_yaml_file(path)
if not cls.is_valid_config(config):
if not cls._is_valid_config(config):
LOG.warning("Skipped invalid config (possible obsoleted): {}"
.format(path))
return []
@ -85,25 +86,56 @@ class StaticDriver(DriverBase):
@classmethod
def _pack(cls, entities, relationships):
entity_index = {}
entities_dict = {}
for entity in entities:
cls._pack_entity(entity_index, entity)
cls._pack_entity(entities_dict, entity)
for rel in relationships:
cls._pack_rel(entity_index, rel)
return entity_index.values()
cls._pack_rel(entities_dict, rel)
return entities_dict.values()
@classmethod
def _pack_entity(cls, entity_index, entity):
config_id = entity[cls.CONFIG_ID]
if config_id not in entity_index:
entity_index[config_id] = entity
def _pack_entity(cls, entities_dict, entity):
config_id = entity[StaticFields.CONFIG_ID]
if config_id not in entities_dict:
metadata = {key: value for key, value in iteritems(entity)
if key not in cls.BASE_FIELDS}
entities_dict[config_id] = entity
entity[TopologyFields.RELATIONSHIPS] = []
entity[TopologyFields.METADATA] = metadata
else:
LOG.warning("Skipped duplicated entity: {}".format(entity))
@classmethod
def _pack_rel(cls, entities_dict, rel):
source_id = rel[TopologyFields.SOURCE]
target_id = rel[TopologyFields.TARGET]
if source_id == target_id:
# self pointing relationship
entities_dict[source_id][TopologyFields.RELATIONSHIPS].append(rel)
else:
source, target = entities_dict[source_id], entities_dict[target_id]
source[TopologyFields.RELATIONSHIPS].append(
cls._expand_neighbor(rel, target))
@staticmethod
def _pack_rel(entity_index, rel):
# use set to handle self pointing relationship
ids = {rel[TopologyFields.SOURCE], rel[TopologyFields.TARGET]}
for config_id in ids:
entity_index[config_id][TopologyFields.RELATIONSHIPS].append(rel)
def _expand_neighbor(rel, neighbor):
"""Expand config id to neighbor entity
rel={'source': 's1', 'target': 'r1', 'relationship_type': 'attached'}
neighbor={'config_id': 'h1', 'type': 'host.nova', 'id': 1}
result={'relationship_type': 'attached', 'source': 's1',
'target': {'config_id': 'h1', 'type': 'host.nova', 'id': 1}}
"""
rel = rel.copy()
if rel[StaticFields.SOURCE] == neighbor[StaticFields.CONFIG_ID]:
rel[StaticFields.SOURCE] = neighbor
elif rel[StaticFields.TARGET] == neighbor[StaticFields.CONFIG_ID]:
rel[StaticFields.TARGET] = neighbor
else:
# TODO(yujunz) raise exception and ignore invalid relationship
LOG.error("Invalid neighbor {} for relationship {}"
.format(neighbor, rel))
return None
return rel

View File

@ -18,10 +18,12 @@ from oslo_log import log as logging
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import TopologyFields
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.resource_transformer_base import \
ResourceTransformerBase
from vitrage.datasources.static import STATIC_DATASOURCE
from vitrage.datasources.static import StaticFields
from vitrage.datasources import transformer_base
import vitrage.graph.utils as graph_utils
@ -29,6 +31,7 @@ LOG = logging.getLogger(__name__)
class StaticTransformer(ResourceTransformerBase):
def __init__(self, transformers, conf):
super(StaticTransformer, self).__init__(transformers, conf)
@ -50,10 +53,13 @@ class StaticTransformer(ResourceTransformerBase):
key_fields = self._key_values(entity_type, entity_id)
return transformer_base.build_key(key_fields)
def get_type(self):
@staticmethod
def get_type():
return STATIC_DATASOURCE
def _create_vertex(self, entity_event):
metadata = entity_event.get(TopologyFields.METADATA, {})
entity_type = entity_event[VProps.TYPE]
entity_id = entity_event[VProps.ID]
sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
@ -63,14 +69,45 @@ class StaticTransformer(ResourceTransformerBase):
state = entity_event[VProps.STATE]
entity_key = self._create_entity_key(entity_event)
return graph_utils.create_vertex(
entity_key,
entity_id=entity_id,
entity_category=EntityCategory.RESOURCE,
entity_type=entity_type,
sample_timestamp=sample_timestamp,
update_timestamp=update_timestamp,
entity_state=state)
# create placeholder for non-static datasource entity
if entity_type in self.transformers:
properties = {
VProps.TYPE: entity_type,
VProps.ID: entity_id,
VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.SAMPLE_TIMESTAMP: sample_timestamp}
return self.create_neighbor_placeholder_vertex(**properties)
else:
return graph_utils.create_vertex(
entity_key,
entity_id=entity_id,
entity_category=EntityCategory.RESOURCE,
entity_type=entity_type,
sample_timestamp=sample_timestamp,
update_timestamp=update_timestamp,
entity_state=state,
metadata=metadata)
def _create_static_neighbor(self, entity_event, rel):
if entity_event[StaticFields.CONFIG_ID] == rel[StaticFields.SOURCE]:
neighbor = rel[StaticFields.TARGET]
is_entity_source = True
elif entity_event[StaticFields.CONFIG_ID] == rel[StaticFields.TARGET]:
neighbor = rel[StaticFields.SOURCE]
is_entity_source = False
else:
LOG.error("Invalid relationship {} in entity_event {}".format(
rel, entity_event))
return None
if rel[StaticFields.SOURCE] == rel[StaticFields.TARGET]:
neighbor = entity_event
return self._create_neighbor(entity_event,
neighbor[VProps.ID],
neighbor[VProps.TYPE],
rel[TopologyFields.RELATIONSHIP_TYPE],
is_entity_source=is_entity_source)
def _create_static_neighbors(self, entity_event):
return []
relationships = entity_event.get(TopologyFields.RELATIONSHIPS, [])
return [self._create_static_neighbor(entity_event, rel)
for rel in relationships]

View File

@ -70,7 +70,7 @@ class StaticPhysicalDriver(DriverBase):
static_entities = []
config = file_utils.load_yaml_file(path)
if StaticDriver.is_valid_config(config):
if StaticDriver._is_valid_config(config):
LOG.warning("Skipped config of new static datasource: {}"
.format(file_))
return []
@ -93,7 +93,7 @@ class StaticPhysicalDriver(DriverBase):
'/' + file_
config = file_utils.load_yaml_file(full_path)
if StaticDriver.is_valid_config(config):
if StaticDriver._is_valid_config(config):
LOG.warning("Skipped config of new static datasource: {}"
.format(file_))
return []

View File

@ -211,7 +211,6 @@ class TransformerBase(object):
VProps.CATEGORY: neighbor_category,
VProps.SAMPLE_TIMESTAMP: sample_timestamp,
self.METADATA: metadata
}
neighbor_vertex = \
self.create_neighbor_placeholder_vertex(**properties)

View File

@ -35,6 +35,7 @@ usage example:
import random
from vitrage.common.constants import DatasourceProperties as DSProps
import vitrage.tests.mocks.trace_generator as tg
@ -317,7 +318,7 @@ def simple_consistency_generators(consistency_num, update_events=0,
def simple_switch_generators(switch_num, host_num,
snapshot_events=0, snap_vals=None,
update_events=0, update_vals=None):
"""A function for returning switch event generators.
"""A function for returning switch events generators.
Returns generators for a given number of switches and hosts.
Hosts will be distributed across switches in round-robin style.
@ -328,8 +329,8 @@ def simple_switch_generators(switch_num, host_num,
:param switch_num: number of zones
:param host_num: number of hosts
:param snapshot_events: number of snapshot events per zone
:param snap_vals: preset vals for ALL snapshot events
:return: generators for zone_num zones as specified
:param snap_vals: preset values for ALL snapshot events
:return: generators for switch events as specified
"""
mapping = [('host-{0}'.format(index), 'switch-{0}'.format(index %
@ -363,6 +364,59 @@ def simple_switch_generators(switch_num, host_num,
return tg.get_trace_generators(test_entity_spec_list)
def simple_static_generators(switch_num=2, host_num=10,
snapshot_events=0, snap_vals=None,
update_events=0, update_vals=None):
"""A function for returning static datasource events generators.
Returns generators for a given number of routers, switches and hosts.
Hosts will be distributed across switches in round-robin style.
Switches are interconnected in a line.
:param switch_num: number of zones
:param host_num: number of hosts
:param snapshot_events: number of snapshot events per zone
:param snap_vals: preset values for ALL snapshot events
:param update_events: number of values from update event
:param update_vals: preset values for update event
:return: generators for static datasource events
"""
# TODO(yujunz) mock routers which connects all switches
mapping = [(host_index, host_index % switch_num)
for host_index in range(host_num)]
test_entity_spec_list = []
if snapshot_events > 0:
if snap_vals is None:
snap_vals = {}
snap_vals[DSProps.DATASOURCE_ACTION] = 'update'
test_entity_spec_list.append(
{tg.DYNAMIC_INFO_FKEY: tg.DRIVER_STATIC_SNAPSHOT_D,
tg.STATIC_INFO_FKEY: None,
tg.EXTERNAL_INFO_KEY: snap_vals,
tg.MAPPING_KEY: mapping,
tg.NAME_KEY: 'Static snapshot generator',
tg.NUM_EVENTS: snapshot_events
}
)
if update_events > 0:
if update_vals is None:
update_vals = {}
update_vals[DSProps.DATASOURCE_ACTION] = 'update'
test_entity_spec_list.append(
{tg.DYNAMIC_INFO_FKEY: tg.DRIVER_STATIC_SNAPSHOT_D,
tg.STATIC_INFO_FKEY: None,
tg.EXTERNAL_INFO_KEY: update_vals,
tg.MAPPING_KEY: mapping,
tg.NAME_KEY: 'Static update generator',
tg.NUM_EVENTS: update_events
}
)
return tg.get_trace_generators(test_entity_spec_list)
def simple_nagios_alarm_generators(host_num,
events_num=0,
snap_vals=None):

View File

@ -22,6 +22,7 @@ multiple instances of the same entity type.
"""
from collections import defaultdict
from random import randint
# noinspection PyPep8Naming
@ -55,6 +56,7 @@ DRIVER_NAGIOS_SNAPSHOT_D = 'driver_nagios_snapshot_dynamic.json'
DRIVER_NAGIOS_SNAPSHOT_S = 'driver_nagios_snapshot_static.json'
DRIVER_ZABBIX_SNAPSHOT_D = 'driver_zabbix_snapshot_dynamic.json'
DRIVER_SWITCH_SNAPSHOT_D = 'driver_switch_snapshot_dynamic.json'
DRIVER_STATIC_SNAPSHOT_D = 'driver_static_snapshot_dynamic.json'
DRIVER_VOLUME_UPDATE_D = 'driver_volume_update_dynamic.json'
DRIVER_VOLUME_SNAPSHOT_D = 'driver_volume_snapshot_dynamic.json'
DRIVER_STACK_UPDATE_D = 'driver_stack_update_dynamic.json'
@ -123,6 +125,7 @@ class EventTraceGenerator(object):
DRIVER_STACK_SNAPSHOT_D: _get_stack_snapshot_driver_values,
DRIVER_STACK_UPDATE_D: _get_stack_update_driver_values,
DRIVER_SWITCH_SNAPSHOT_D: _get_switch_snapshot_driver_values,
DRIVER_STATIC_SNAPSHOT_D: _get_static_snapshot_driver_values,
DRIVER_NAGIOS_SNAPSHOT_D: _get_nagios_alarm_driver_values,
DRIVER_ZABBIX_SNAPSHOT_D: _get_zabbix_alarm_driver_values,
DRIVER_CONSISTENCY_UPDATE_D:
@ -534,6 +537,94 @@ def _get_switch_snapshot_driver_values(spec):
return static_values
def _get_static_snapshot_driver_values(spec):
"""Generates the static driver values for static datasource.
:param spec: specification of event generation.
:type spec: dict
:return: list of driver values for static datasource.
:rtype: list
"""
host_switch_mapping = spec[MAPPING_KEY]
static_info_spec = None
if spec[STATIC_INFO_FKEY] is not None:
static_info_spec = utils.load_specs(spec[STATIC_INFO_FKEY])
static_values = []
relationships = defaultdict(lambda: [])
entities = defaultdict(lambda: {})
for host_index, switch_index in host_switch_mapping:
host_id = "h{}".format(host_index)
switch_id = "s{}".format(switch_index)
relationship = {
"source": switch_id,
"target": host_id,
"relationship_type": "attached"
}
host_rel = relationship.copy()
host_rel['source'] = entities[switch_id]
relationships[host_id].append(host_rel)
switch_rel = relationship.copy()
switch_rel['target'] = entities[host_id]
relationships[switch_id].append(switch_rel)
for host_index, switch_index in host_switch_mapping:
mapping = {}
switch_id = "s{}".format(switch_index)
if switch_id not in entities:
switch_name = "switch-{}".format(switch_index)
mapping = {
'name': switch_name,
'config_id': switch_id,
'id': str(randint(0, 100000)),
'type': 'switch',
'relationships': relationships[switch_id]
}
entities[switch_id].update(**mapping)
host_id = "h{}".format(host_index)
if host_id not in entities:
mapping = {
'config_id': host_id,
'type': 'nova.host',
'id': str(randint(0, 100000)),
'relationships': relationships[host_id]
}
entities[host_id].update(**mapping)
static_values.append(combine_data(static_info_spec,
mapping,
spec.get(EXTERNAL_INFO_KEY, None)))
for index in range(10):
custom_id = 'c{}'.format(index)
# self-pointing relationship
relationships = [
{
"source": custom_id,
"target": custom_id,
"relationship_type": "custom"
}
]
mapping = {
'config_id': custom_id,
'type': 'custom',
'id': str(randint(0, 100000)),
'relationships': relationships
}
static_values.append(combine_data(static_info_spec,
mapping,
spec.get(EXTERNAL_INFO_KEY, None)))
return static_values
def _get_nagios_alarm_driver_values(spec):
hosts = spec[MAPPING_KEY]
static_info_re = None

View File

@ -0,0 +1,17 @@
{
"name": "[switch|host|custom]-[1-9]",
"id": "[1-9]{5}",
"config_id": "[shc][1-9]",
"type": "[switch|nova.host|custom]",
"state": "available",
"vitrage_datasource_action": "snapshot",
"vitrage_sample_date": "2015-12-01T12:46:41Z",
"vitrage_event_type": "entity_update",
"relationships": [
{
"source": "[sc][1-9]",
"target": "[hc][1-9]",
"relationship_type": "attached|custom"
}
]
}

View File

@ -37,28 +37,28 @@ definitions:
relationships:
- source: s1
target: r1
relation_type: attached
relationship_type: attached
- source: s2
target: r1
relation_type: attached
relationship_type: attached
- source: s3
target: r1
relation_type: attached
relationship_type: attached
- source: s1
target: h1
relation_type: attached
relationship_type: attached
- source: s1
target: h2
relation_type: attached
relationship_type: attached
- source: s2
target: h2
relation_type: attached
relationship_type: attached
- source: s2
target: h3
relation_type: attached
relationship_type: attached
- source: s3
target: h3
relation_type: attached
relationship_type: attached
- source: s3
target: h4
relation_type: attached
relationship_type: attached

View File

@ -20,6 +20,7 @@ from vitrage.common.constants import GraphAction
from vitrage.common.constants import TopologyFields
from vitrage.datasources.static import driver
from vitrage.datasources.static import STATIC_DATASOURCE
from vitrage.datasources.static import StaticFields
from vitrage.tests import base
from vitrage.tests.mocks import utils
@ -78,7 +79,7 @@ class TestStaticDriver(base.BaseTest):
self.assertEqual(9, len(static_entities))
for entity in static_entities[:-1]: # exclude end message
self.assert_is_not_empty(entity[TopologyFields.RELATIONSHIPS])
self._validate_static_entity(entity)
# noinspection PyAttributeOutsideInit
def test_get_changes(self):
@ -98,4 +99,20 @@ class TestStaticDriver(base.BaseTest):
# Test Assertions
self.assertEqual(0, len(changes))
for entity in changes:
self.assert_is_not_empty(entity[TopologyFields.RELATIONSHIPS])
self._validate_static_entity(entity)
def _validate_static_entity(self, entity):
self.assertTrue(isinstance(entity[TopologyFields.METADATA], dict))
for rel in entity[TopologyFields.RELATIONSHIPS]:
self._validate_static_rel(entity, rel)
def _validate_static_rel(self, entity, rel):
self.assertTrue(entity[StaticFields.CONFIG_ID] in
(rel[StaticFields.SOURCE], rel[StaticFields.TARGET]))
self.assertTrue(
isinstance(rel[StaticFields.SOURCE], dict)
and entity[StaticFields.CONFIG_ID] == rel[StaticFields.TARGET]
or isinstance(rel[StaticFields.TARGET], dict)
and entity[StaticFields.CONFIG_ID] == rel[StaticFields.SOURCE]
or entity[StaticFields.CONFIG_ID] == rel[StaticFields.SOURCE]
and entity[StaticFields.CONFIG_ID] == rel[StaticFields.TARGET])

View File

@ -12,13 +12,21 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_config import cfg
from oslo_log import log as logging
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import UpdateMethod
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE
from vitrage.datasources.nova.host.transformer import HostTransformer
from vitrage.datasources.static import STATIC_DATASOURCE
from vitrage.datasources.static.transformer import StaticTransformer
from vitrage.tests import base
from vitrage.tests.mocks import mock_driver
LOG = logging.getLogger(__name__)
@ -36,11 +44,90 @@ class TestStaticTransformer(base.BaseTest):
cls.transformers = {}
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group=STATIC_DATASOURCE)
cls.transformers[STATIC_DATASOURCE] = \
StaticTransformer(cls.transformers, cls.conf)
cls.transformer = StaticTransformer(cls.transformers, cls.conf)
cls.transformers[STATIC_DATASOURCE] = cls.transformer
cls.transformers[NOVA_HOST_DATASOURCE] = \
HostTransformer(cls.transformers, cls.conf)
# noinspection PyAttributeOutsideInit
def setUp(self):
super(TestStaticTransformer, self).setUp()
self.entity_type = STATIC_DATASOURCE
self.entity_id = '12345'
self.timestamp = datetime.datetime.utcnow()
def test_create_placeholder_vertex(self):
properties = {
VProps.TYPE: self.entity_type,
VProps.ID: self.entity_id,
VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.SAMPLE_TIMESTAMP: self.timestamp
}
placeholder = self.transformer.create_neighbor_placeholder_vertex(
**properties)
observed_entity_id = placeholder.vertex_id
expected_entity_id = 'RESOURCE:static:12345'
self.assertEqual(observed_entity_id, expected_entity_id)
observed_time = placeholder.get(VProps.SAMPLE_TIMESTAMP)
self.assertEqual(observed_time, self.timestamp)
observed_subtype = placeholder.get(VProps.TYPE)
self.assertEqual(observed_subtype, self.entity_type)
observed_entity_id = placeholder.get(VProps.ID)
self.assertEqual(observed_entity_id, self.entity_id)
observed_category = placeholder.get(VProps.CATEGORY)
self.assertEqual(observed_category, EntityCategory.RESOURCE)
is_placeholder = placeholder.get(VProps.IS_PLACEHOLDER)
self.assertEqual(is_placeholder, True)
def test_snapshot_transform(self):
pass
spec_list = mock_driver.simple_static_generators(snapshot_events=10)
events = mock_driver.generate_random_events_list(spec_list)
self._event_transform_test(events)
def test_update_transform(self):
spec_list = mock_driver.simple_static_generators(update_events=10)
events = mock_driver.generate_random_events_list(spec_list)
self._event_transform_test(events)
def _event_transform_test(self, events):
for event in events:
wrapper = self.transformer.transform(event)
vertex = wrapper.vertex
self._validate_vertex(vertex, event)
neighbors = wrapper.neighbors
self._validate_neighbors(neighbors, vertex.vertex_id, event)
def _validate_vertex(self, vertex, event):
self.assertEqual(vertex[VProps.CATEGORY], EntityCategory.RESOURCE)
self.assertEqual(vertex[VProps.TYPE], event[VProps.TYPE])
self.assertEqual(vertex[VProps.ID], event[VProps.ID])
self.assertEqual(vertex[VProps.SAMPLE_TIMESTAMP],
event[DSProps.SAMPLE_DATE])
self.assertFalse(vertex[VProps.IS_DELETED])
def _validate_neighbors(self, neighbors, vertex_id, event):
for i in range(len(neighbors)):
if event['type'] == 'nova.host':
self._validate_switch_neighbor(neighbors[i],
event['relationships'][i],
vertex_id)
elif event['type'] == 'switch':
self._validate_host_neighbor(neighbors[i],
event['relationships'][i],
vertex_id)
def _validate_switch_neighbor(self, neighbor, rel, host_vertex_id):
# TODO(yujunz)
pass
def _validate_host_neighbor(self, neighbor, rel, switch_vertex_id):
# TODO(yujunz)
pass

View File

@ -62,7 +62,7 @@ class BaseAlarmTransformerTest(BaseTransformerTest):
VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.SAMPLE_TIMESTAMP: wrapper.vertex[VProps.SAMPLE_TIMESTAMP],
}
expected_neighbor = host_transformer. \
expected_neighbor = host_transformer.\
create_neighbor_placeholder_vertex(**properties)
self.assertEqual(expected_neighbor, host_neighbor.vertex)