openstack/vitrage
Code Issues Proposed changes

Merge "Remove the static physical datasource"

Browse Source
This commit is contained in:
Zuul 2018-12-13 10:49:17 +00:00 committed by Gerrit Code Review
commit dc48a3ecfa
20 changed files with 25 additions and 1048 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
devstack/gate_hook.sh
View File

@ -64,9 +64,6 @@ topics = notifications, vitrage_notifications
[DEFAULT]
notifiers = mistral,nova,webhook
[static_physical]
changes_interval = 5
[datasources]
snapshots_interval = 120
EOF

2
devstack/settings
View File

@ -37,7 +37,7 @@ VITRAGE_DEPLOY=${VITRAGE_DEPLOY}
# Toggle for deploying Vitrage with/without nagios
VITRAGE_USE_NAGIOS=$(trueorfalse False VITRAGE_USE_NAGIOS)
VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,static_physical,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus}
VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus}
# for now dont use pip install for the client
LIBS_FROM_GIT=python-vitrageclient

1
doc/source/contributor/configuration.rst
View File

@ -24,7 +24,6 @@ Datasources
nagios-config
static-config
static-physical-config
zabbix_vitrage
k8s_datasource

30
doc/source/contributor/static-config.rst
View File

@ -5,14 +5,15 @@ Static Datasource Configuration
Overview
--------
The static datasource allows users to integrate the **unmanaged** resources and topology into Vitrage. Unmanaged means
the resource, relationship or property can not be retrieved from any API or database, except static configuration file.
The static datasource allows users to integrate **unmanaged** resources and
topology into Vitrage. Unmanaged means the resource, relationship or property
can not be retrieved from any API or database, except static configuration
file. The static configuration may include physical, virtual or application
resources.
This datasource is static - pre-configured in a file. This is sufficient in many cases where the resources and
relationship is relatively unchanging.
Static datasource suppresses the legacy static physical datasource. Theoretically both physical and virtual resources
and relationship between them can be configured in it.
This datasource is static. It is configured in a file that is being reloaded
periodically, based on the configuration. This is sufficient in many cases
where the resources and relationship is relatively unchanging.
Configure Access to Static
--------------------------
@ -20,13 +21,13 @@ Configure Access to Static
The following should be set in **/etc/vitrage/vitrage.conf**, under
``[static]`` section:
+------------------+---------------------------------------------------------+----------------------------------+
+------------------+--------------------------------------------------------+----------------------------------+
| Name | Description | Default Value |
+==================+=========================================================+==================================+
+==================+========================================================+==================================+
| directory | Directory path from where to load the configurations | /etc/vitrage/static_datasources/ |
+------------------+---------------------------------------------------------+----------------------------------+
| changes_interval | Interval of checking changes in the configuration files | 30 seconds |
+------------------+---------------------------------------------------------+----------------------------------+
+------------------+--------------------------------------------------------+----------------------------------+
| changes_interval | Interval of checking changes in the configuration file | 30 seconds |
+------------------+--------------------------------------------------------+----------------------------------+
Configure Static Mapping
------------------------
@ -45,11 +46,6 @@ Static datasource use the same semantics as Vitrage template, except for the fol
There may be more than one configuration file. All files will be read from ``/etc/vitrage/static_datasources/``. See
previous section on how to configure this location.
Notes:
- Static datasource shares the same configuration folder as legacy static physical datasource.
- Both static configuration and legacy static physical configuration will be loaded in Ocata release.
- The format is distinguished by checking existence of ``metadata`` key which is only available in static datasource.
Example
+++++++

98
doc/source/contributor/static-physical-config.rst
View File

@ -1,98 +0,0 @@
====================================================
DEPRECATED: static physical datasource configuration
====================================================
**The static physical datasource is deprecated and will be removed in Stein
release.**
**Please use the static datasource instead.**
Overview
--------
The Static Physical datasource allows users to integrate the physical topology
into Vitrage. Physical topology includes switches and their connection to
other switches and physical hosts.
This datasource is static - pre-configured in a file. This is sufficient in
many cases where the physical topology is relatively unchanging.
Configure Access to Static Physical
-----------------------------------
The following should be set in **/etc/vitrage/vitrage.conf**, under
[static_physical] section:
+------------------+---------------------------------------------------------+----------------------------------+
| Name | Description | Default Value |
+==================+=========================================================+==================================+
| directory | Directory path from where to load the configurations | /etc/vitrage/static_datasources/ |
+------------------+---------------------------------------------------------+----------------------------------+
| changes_interval | Interval of checking changes in the configuration files | 30 seconds |
+------------------+---------------------------------------------------------+----------------------------------+
| entities | Static physical entity types list | switch |
+------------------+---------------------------------------------------------+----------------------------------+
Configure Static Physical Mapping
---------------------------------
Physical configuration is made for configuring statically physical entities,
and their relationships to other entities in the topology.
Some physical entities, such as switches, can not be retrieved from OpenStack,
and so are defined here.
There may be more than one configuration file. All files will be read from
*/etc/vitrage/static_datasources/*. See previous section on how to configure this
location.
Format
++++++
.. code::
entities:
- name: <Physical entity name as appears in configuration>
id: <Physical entity id as appears in configuration>
type: <Physical entity type - see below for details>
state: <default resource state>
relationships:
- type: <Physical entity type it is connected to - see below for details>
name: <Name of physical entity as appears in configuration>
id: <Id of physical entity as appears in configuration>
relation_type: <Relation name>
- type: ...
...
Notes:
- The "type" key must match the name of a type from an existing datasource.
- Type names appear, for each datasource, in its __init__.py file.
- For example see */workspace/dev/vitrage/vitrage/datasources/nova/host/__init__.py*
Example
+++++++
The following will define a switch that is attached to host-1 and is a backup
of switch-2
.. code::
entities:
- type: switch
name: switch-1
id: switch-1 # should be same as name
state: available
relationships:
- type: nova.host
name: host-1
id: host-1 # should be same as name
relation_type: attached
- type: switch
name: switch-2
id: switch-2 # should be same as name
relation_type: backup

4
releasenotes/notes/remove-static-physical-datasource-e8e70158f453cff1.yaml Normal file
View File

@ -0,0 +1,4 @@
---
deprecations:
- The static_physical datasource was removed. Please use the static
datasource instead.

3
vitrage/datasources/static/__init__.py
View File

@ -41,9 +41,6 @@ OPTS = [
min=10,
help='interval in seconds between checking changes in the'
'static configuration files'),
# NOTE: This folder is already used by static_physical datasource. Legacy
# configuration files will NOT be converted automatically. But user will
# receive deprecation warnings.
cfg.StrOpt('directory', default='/etc/vitrage/static_datasources',
help='static data sources configuration directory')]

75
vitrage/datasources/static_physical/__init__.py
View File

@ -1,75 +0,0 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_log import versionutils
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import UpdateMethod
from vitrage.common import utils
STATIC_PHYSICAL_DATASOURCE = 'static_physical'
SWITCH = 'switch'
_DEPRECATED_MSG = utils.fmt("""
`Static_physical` was deprecated in Queens and and will be removed in Stein.
Please use static Datasource.
""")
OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.static_physical.transformer.'
'StaticPhysicalTransformer',
help='Static physical transformer class path',
deprecated_for_removal=True,
deprecated_reason=_DEPRECATED_MSG,
deprecated_since=versionutils.deprecated.QUEENS,
required=True),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.static_physical.driver.'
'StaticPhysicalDriver',
help='Static physical driver class path',
required=True),
cfg.StrOpt(DSOpts.UPDATE_METHOD,
default=UpdateMethod.PULL,
help='None: updates only via Vitrage periodic snapshots.'
'Pull: updates every [changes_interval] seconds.'
'Push: updates by getting notifications from the'
' datasource itself.',
deprecated_for_removal=True,
deprecated_reason=_DEPRECATED_MSG,
deprecated_since=versionutils.deprecated.QUEENS,
required=True),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=20,
min=5,
help='interval between checking changes in the configuration '
'files of the physical topology data sources',
deprecated_for_removal=True,
deprecated_reason=_DEPRECATED_MSG,
deprecated_since=versionutils.deprecated.QUEENS),
cfg.StrOpt('directory', default='/etc/vitrage/static_datasources',
help='Static physical data sources directory',
deprecated_for_removal=True,
deprecated_reason=_DEPRECATED_MSG,
deprecated_since=versionutils.deprecated.QUEENS),
cfg.ListOpt('entities',
default=[SWITCH],
help='Static physical entity types list',
deprecated_for_removal=True,
deprecated_reason=_DEPRECATED_MSG,
deprecated_since=versionutils.deprecated.QUEENS)
]

167
vitrage/datasources/static_physical/driver.py
View File

@ -1,167 +0,0 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
from debtcollector import removals
from oslo_log import log
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import GraphAction
from vitrage.datasources.driver_base import DriverBase
from vitrage.datasources.static.driver import StaticDriver
from vitrage.datasources.static import StaticFields
from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE
from vitrage.utils import file as file_utils
LOG = log.getLogger(__name__)
class StaticPhysicalDriver(DriverBase):
@staticmethod
def get_event_types():
return []
def enrich_event(self, event, event_type):
pass
ENTITIES_SECTION = 'entities'
def __init__(self, conf):
removals.removed_module(__name__, "datasources.static")
super(StaticPhysicalDriver, self).__init__()
self.cfg = conf
self.cache = {}
def get_all(self, datasource_action):
return self.make_pickleable(self._get_all_entities(),
STATIC_PHYSICAL_DATASOURCE,
datasource_action)
def get_changes(self, datasource_action):
return self.make_pickleable(self._get_changes_entities(),
STATIC_PHYSICAL_DATASOURCE,
datasource_action)
def _get_all_entities(self):
static_entities = []
files = file_utils.list_files(
self.cfg.static_physical.directory, '.yaml')
for file_ in files:
full_path = self.cfg.static_physical.directory \
+ '/' + file_
static_entities += self._get_entities_from_file(file_,
full_path)
return static_entities
def _get_entities_from_file(self, file_, path):
static_entities = []
config = file_utils.load_yaml_file(path)
if StaticDriver._is_valid_config(config):
LOG.warning("Skipped config of new static datasource: {}"
.format(file_))
return []
for entity in config[self.ENTITIES_SECTION]:
static_entities.append(entity.copy())
self.cache[file_] = config
return static_entities
def _get_changes_entities(self):
entities_updates = []
files = file_utils.list_files(
self.cfg.static_physical.directory, '.yaml')
for file_ in files:
full_path = self.cfg.static_physical.directory +\
'/' + file_
config = file_utils.load_yaml_file(full_path)
if StaticDriver._is_valid_config(config):
LOG.warning("Skipped config of new static datasource: {}"
.format(file_))
return []
if config:
if file_ in self.cache:
if str(config) != str(self.cache[file_]):
# TODO(alexey_weyl): need also to remove deleted
# files from cache
old_config = copy.deepcopy(config)
self._update_on_existing_entities(
self.cache[file_][self.ENTITIES_SECTION],
config[self.ENTITIES_SECTION],
entities_updates)
self._update_on_new_entities(
config[self.ENTITIES_SECTION],
self.cache[file_][self.ENTITIES_SECTION],
entities_updates)
self.cache[file_] = old_config
else:
self.cache[file_] = config
entities_updates += \
self._get_entities_from_file(file_, full_path)
# iterate over deleted files
deleted_files = set(self.cache.keys()) - set(files)
for file_ in deleted_files:
self._update_on_existing_entities(
self.cache[file_][self.ENTITIES_SECTION],
{},
entities_updates)
del self.cache[file_]
return entities_updates
def _update_on_existing_entities(self,
old_entities,
new_entities,
updates):
for old_entity in old_entities:
if not new_entities or old_entity not in new_entities:
new_entity = self._find_entity(old_entity, new_entities)
if not new_entity:
self._set_event_type(old_entity, GraphAction.DELETE_ENTITY)
updates.append(old_entity.copy())
else:
self._set_event_type(new_entity, GraphAction.UPDATE_ENTITY)
updates.append(new_entity.copy())
@staticmethod
def _find_entity(new_entity, entities):
for entity in entities:
if entity[StaticFields.TYPE] == new_entity[StaticFields.TYPE] \
and entity[StaticFields.ID] == new_entity[StaticFields.ID]:
return entity
return None
@staticmethod
def _update_on_new_entities(new_entities, old_entities, updates):
for entity in new_entities:
if entity not in updates and entity not in old_entities:
updates.append(entity.copy())
@staticmethod
def _set_event_type(entity, event_type):
entity[DSProps.EVENT_TYPE] = event_type

128
vitrage/datasources/static_physical/transformer.py
View File

@ -1,128 +0,0 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from debtcollector import removals
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import GraphAction
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE
from vitrage.datasources.resource_transformer_base import \
ResourceTransformerBase
from vitrage.datasources.static import StaticFields
from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE
from vitrage.datasources.static_physical import SWITCH
from vitrage.datasources import transformer_base
import vitrage.graph.utils as graph_utils
class StaticPhysicalTransformer(ResourceTransformerBase):
RELATION_TYPE = 'relation_type'
RELATIONSHIPS_SECTION = 'relationships'
# graph actions which need to refer them differently
GRAPH_ACTION_MAPPING = {
GraphAction.DELETE_ENTITY: GraphAction.DELETE_ENTITY
}
def __init__(self, transformers, conf):
removals.removed_module(__name__, "datasources.static")
super(StaticPhysicalTransformer, self).__init__(transformers, conf)
self._register_relations_direction()
def _create_snapshot_entity_vertex(self, entity_event):
return self._create_vertex(entity_event)
def _create_update_entity_vertex(self, entity_event):
return self._create_vertex(entity_event)
def _create_vertex(self, entity_event):
entity_type = entity_event[StaticFields.TYPE]
entity_id = entity_event[VProps.ID]
vitrage_sample_timestamp = entity_event[DSProps.SAMPLE_DATE]
update_timestamp = self._format_update_timestamp(
update_timestamp=None,
sample_timestamp=vitrage_sample_timestamp)
state = entity_event[VProps.STATE]
entity_key = self._create_entity_key(entity_event)
metadata = self._extract_metadata(entity_event)
return graph_utils.create_vertex(
entity_key,
vitrage_category=EntityCategory.RESOURCE,
vitrage_type=entity_type,
vitrage_sample_timestamp=vitrage_sample_timestamp,
entity_id=entity_id,
update_timestamp=update_timestamp,
entity_state=state,
metadata=metadata)
def _create_snapshot_neighbors(self, entity_event):
return self._create_static_physical_neighbors(entity_event)
def _create_update_neighbors(self, entity_event):
return self._create_static_physical_neighbors(entity_event)
def _create_static_physical_neighbors(self, entity_event):
neighbors = []
entity_type = entity_event[StaticFields.TYPE]
for neighbor_details in entity_event.get(
self.RELATIONSHIPS_SECTION, {}):
# TODO(alexey): need to decide what to do if one of the entities
# fails
neighbor_id = neighbor_details[VProps.ID]
neighbor_type = neighbor_details[StaticFields.TYPE]
relation_type = neighbor_details[self.RELATION_TYPE]
is_entity_source = not self._find_relation_direction_source(
entity_type, neighbor_type)
neighbor = self._create_neighbor(entity_event,
neighbor_id,
neighbor_type,
relation_type,
is_entity_source=is_entity_source)
if neighbor is not None:
neighbors.append(neighbor)
return neighbors
def _create_entity_key(self, entity_event):
entity_id = entity_event[VProps.ID]
entity_type = entity_event[StaticFields.TYPE]
key_fields = self._key_values(entity_type, entity_id)
return transformer_base.build_key(key_fields)
@staticmethod
def _extract_metadata(entity_event):
metadata = {VProps.NAME: entity_event[VProps.NAME]}
return metadata
def _find_relation_direction_source(self, entity_type, neighbor_type):
# TODO(alexey): maybe check if this type exists, because it throws
# exception if it doesn't
return self.relation_direction[(entity_type, neighbor_type)]
def _register_relations_direction(self):
self.relation_direction = {}
relationship = (SWITCH, NOVA_HOST_DATASOURCE)
self.relation_direction[relationship] = True
relationship = (SWITCH, SWITCH)
self.relation_direction[relationship] = True
def get_vitrage_type(self):
return STATIC_PHYSICAL_DATASOURCE

0
vitrage/tests/functional/datasources/static_physical/__init__.py
View File

99
vitrage/tests/functional/datasources/static_physical/test_static_physical.py
View File

@ -1,99 +0,0 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from testtools import matchers
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.nagios import NAGIOS_DATASOURCE
from vitrage.datasources import NOVA_HOST_DATASOURCE
from vitrage.datasources import NOVA_INSTANCE_DATASOURCE
from vitrage.datasources import NOVA_ZONE_DATASOURCE
from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE
from vitrage.datasources.static_physical import SWITCH
from vitrage.tests.functional.datasources.base import \
TestDataSourcesBase
from vitrage.tests.mocks import mock_driver
class TestStaticPhysical(TestDataSourcesBase):
DATASOURCES_OPTS = [
cfg.ListOpt('types',
default=[NAGIOS_DATASOURCE,
NOVA_HOST_DATASOURCE,
NOVA_INSTANCE_DATASOURCE,
NOVA_ZONE_DATASOURCE,
STATIC_PHYSICAL_DATASOURCE],
help='Names of supported driver data sources'),
cfg.ListOpt('path',
default=['vitrage.datasources'],
help='base path for data sources')
]
# noinspection PyPep8Naming
@classmethod
def setUpClass(cls):
super(TestStaticPhysical, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph')
cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources')
cls.load_datasources(cls.conf)
def test_static_physical_validity(self):
# Setup
processor = self._create_processor_with_graph(self.conf)
transformers = processor.transformer_manager.transformers
transformers[SWITCH] = transformers[STATIC_PHYSICAL_DATASOURCE]
self.assertThat(processor.entity_graph,
matchers.HasLength(
self._num_total_expected_vertices())
)
spec_list = mock_driver.simple_switch_generators(
switch_num=1,
host_num=1,
snapshot_events=1)
static_events = mock_driver.generate_random_events_list(spec_list)
static_physical_event = static_events[0]
static_physical_event[DSProps.ENTITY_TYPE] = SWITCH
static_physical_event['relationships'][0]['name'] = \
self._find_entity_id_by_type(processor.entity_graph,
NOVA_HOST_DATASOURCE)
# Action
processor.process_event(static_physical_event)
# Test assertions
self.assertThat(processor.entity_graph,
matchers.HasLength(
self._num_total_expected_vertices() + 1)
)
static_physical_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter={
VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE,
VProps.VITRAGE_TYPE: SWITCH
})
self.assertThat(static_physical_vertices, matchers.HasLength(1))
static_physical_neighbors = processor.entity_graph.neighbors(
static_physical_vertices[0].vertex_id)
self.assertThat(static_physical_neighbors, matchers.HasLength(1))
self.assertEqual(NOVA_HOST_DATASOURCE,
static_physical_neighbors[0][VProps.VITRAGE_TYPE])

50
vitrage/tests/mocks/mock_driver.py
View File

@ -317,56 +317,6 @@ def simple_consistency_generators(consistency_num, update_events=0,
return tg.get_trace_generators(test_entity_spec_list)
def simple_switch_generators(switch_num, host_num,
snapshot_events=0, snap_vals=None,
update_events=0, update_vals=None):
"""A function for returning switch events generators.
Returns generators for a given number of switches and hosts.
Hosts will be distributed across switches in round-robin style.
Switches are interconnected in a line.
:param update_vals: number of events from update event
:param update_events: number of values from update event
:param switch_num: number of zones
:param host_num: number of hosts
:param snapshot_events: number of snapshot events per zone
:param snap_vals: preset values for ALL snapshot events
:return: generators for switch events as specified
"""
mapping = [('host-{0}'.format(index), 'switch-{0}'.format(index %
switch_num))
for index in range(host_num)
]
test_entity_spec_list = []
if snapshot_events:
test_entity_spec_list.append(
{tg.DYNAMIC_INFO_FKEY: tg.DRIVER_SWITCH_SNAPSHOT_D,
tg.STATIC_INFO_FKEY: None,
tg.EXTERNAL_INFO_KEY: snap_vals,
tg.MAPPING_KEY: mapping,
tg.NAME_KEY: 'Switch snapshot generator',
tg.NUM_EVENTS: snapshot_events
}
)
if update_events:
update_vals = {} if not update_vals else update_vals
update_vals[DSProps.DATASOURCE_ACTION] = \
DatasourceAction.UPDATE
test_entity_spec_list.append(
{tg.DYNAMIC_INFO_FKEY: tg.DRIVER_SWITCH_SNAPSHOT_D,
tg.STATIC_INFO_FKEY: None,
tg.EXTERNAL_INFO_KEY: update_vals,
tg.MAPPING_KEY: mapping,
tg.NAME_KEY: 'Switch update generator',
tg.NUM_EVENTS: update_events
}
)
return tg.get_trace_generators(test_entity_spec_list)
def simple_static_generators(switch_num=2, host_num=10,
snapshot_events=0, snap_vals=None,
update_events=0, update_vals=None):

42
vitrage/tests/mocks/trace_generator.py
View File

@ -58,7 +58,6 @@ DRIVER_NAGIOS_SNAPSHOT_D = 'driver_nagios_snapshot_dynamic.json'
DRIVER_NAGIOS_SNAPSHOT_S = 'driver_nagios_snapshot_static.json'
DRIVER_PROMETHEUS_UPDATE_D = 'driver_prometheus_update_dynamic.json'
DRIVER_ZABBIX_SNAPSHOT_D = 'driver_zabbix_snapshot_dynamic.json'
DRIVER_SWITCH_SNAPSHOT_D = 'driver_switch_snapshot_dynamic.json'
DRIVER_STATIC_SNAPSHOT_D = 'driver_static_snapshot_dynamic.json'
DRIVER_STATIC_SNAPSHOT_S = 'driver_static_snapshot_static.json'
DRIVER_VOLUME_UPDATE_D = 'driver_volume_update_dynamic.json'
@ -131,7 +130,6 @@ class EventTraceGenerator(object):
DRIVER_VOLUME_UPDATE_D: _get_volume_update_driver_values,
DRIVER_STACK_SNAPSHOT_D: _get_stack_snapshot_driver_values,
DRIVER_STACK_UPDATE_D: _get_stack_update_driver_values,
DRIVER_SWITCH_SNAPSHOT_D: _get_switch_snapshot_driver_values,
DRIVER_STATIC_SNAPSHOT_D: _get_static_snapshot_driver_values,
DRIVER_NAGIOS_SNAPSHOT_D: _get_nagios_alarm_driver_values,
DRIVER_ZABBIX_SNAPSHOT_D: _get_zabbix_alarm_driver_values,
@ -513,46 +511,6 @@ def _get_vm_update_driver_values(spec):
return static_values
def _get_switch_snapshot_driver_values(spec):
"""Generates the static driver values for each zone.
:param spec: specification of event generation.
:type spec: dict
:return: list of static driver values for each zone.
:rtype: list
"""
host_switch_mapping = spec[MAPPING_KEY]
static_info = None
if spec[STATIC_INFO_FKEY] is not None:
static_info = utils.load_specs(spec[STATIC_INFO_FKEY])
static_values = []
switches_info = {}
for host_name, switch_name in host_switch_mapping:
switch_info = switches_info.get(switch_name, [])
relationship_info = {"type": NOVA_HOST_DATASOURCE,
"name": host_name,
"id": host_name,
"relation_type": "contains"
}
switch_info.append(relationship_info)
switches_info[switch_name] = switch_info
for host_name, switch_name in host_switch_mapping:
mapping = {'name': switch_name,
'id': switch_name,
'relationships': switches_info[switch_name]
}
static_values.append(combine_data(static_info,
mapping,
spec.get(EXTERNAL_INFO_KEY, None)))
return static_values
def _get_static_snapshot_driver_values(spec):
"""Generates the static driver values for static datasource.

16
vitrage/tests/resources/mock_configurations/driver/driver_switch_snapshot_dynamic.json
View File

@ -1,16 +0,0 @@
{
"relationships": [
{"type": "nova.host",
"name": "host-[1-9]",
"id": "[1-9]{5}",
"relation_type": "contains"
}
],
"name": "switch-[0-9]",
"id": "[1-9]{5}",
"vitrage_sample_date": "2015-12-01T12:46:41Z",
"type": "switch",
"vitrage_datasource_action": "snapshot",
"state": "available",
"vitrage_event_type": "entity_update"
}

0
vitrage/tests/unit/datasources/static_physical/__init__.py
View File

145
vitrage/tests/unit/datasources/static_physical/test_static_physical_driver.py
View File

@ -1,145 +0,0 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from oslo_config import cfg
from testtools import matchers
from vitrage.common.constants import DatasourceAction
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import GraphAction
from vitrage.datasources.static import StaticFields
from vitrage.datasources.static_physical import driver
from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE
from vitrage.datasources.static_physical import SWITCH
from vitrage.tests import base
from vitrage.tests.base import IsEmpty
from vitrage.tests.mocks import utils
from vitrage.utils import file as file_utils
class TestStaticPhysicalDriver(base.BaseTest):
OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.static_physical.transformer.'
'StaticPhysicalTransformer'),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.static_physical.driver.'
'StaticPhysicalDriver'),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=30,
help='interval between checking changes in the '
'configuration files of the physical topology plugin'),
cfg.StrOpt('directory',
default=utils.get_resources_dir() + '/static_datasources'),
cfg.ListOpt('entities',
default=[SWITCH])
]
CHANGES_OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.static_physical.transformer.'
'StaticPhysicalTransformer'),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.static_physical.driver.'
'StaticPhysicalDriver'),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=30,
help='interval between checking changes in the '
'configuration files of the physical topology plugin'),
cfg.StrOpt('directory',
default=utils.get_resources_dir() +
'/static_datasources/changes_datasources'),
]
# noinspection PyAttributeOutsideInit,PyPep8Naming
@classmethod
def setUpClass(cls):
super(TestStaticPhysicalDriver, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group=STATIC_PHYSICAL_DATASOURCE)
cls.static_physical_driver = driver.StaticPhysicalDriver(cls.conf)
def test_static_datasources_loader(self):
# Setup
total_static_datasources = \
os.listdir(self.conf.static_physical.directory)
# Action
static_configs = file_utils.load_yaml_files(
self.conf.static_physical.directory)
# Test assertions
# -1 is because there are 2 files and a folder in static_datasource_dir
self.assertEqual(len(total_static_datasources) - 1,
len(static_configs))
def test_get_all(self):
# Action
static_entities = \
self.static_physical_driver.get_all(DatasourceAction.UPDATE)
# Test assertions
self.assertThat(static_entities, matchers.HasLength(5))
# noinspection PyAttributeOutsideInit
def test_get_changes(self):
# Setup
entities = self.static_physical_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(5))
self.conf = cfg.ConfigOpts()
self.conf.register_opts(self.CHANGES_OPTS,
group=STATIC_PHYSICAL_DATASOURCE)
self.static_physical_driver.cfg = self.conf
# Action
changes = self.static_physical_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
status = any(change[StaticFields.TYPE] == SWITCH and
change[StaticFields.ID] == '12345' for change in changes)
self.assertFalse(status)
status = any(change[StaticFields.TYPE] == SWITCH and
change[StaticFields.ID] == '23456' and
change[DSProps.EVENT_TYPE] == GraphAction.DELETE_ENTITY
for change in changes)
self.assertTrue(status)
status = any(change[StaticFields.TYPE] == SWITCH and
change[StaticFields.ID] == '34567' for change in changes)
self.assertTrue(status)
status = any(change[StaticFields.TYPE] == SWITCH and
change[StaticFields.ID] == '45678' for change in changes)
self.assertTrue(status)
status = any(change[StaticFields.TYPE] == SWITCH and
change[StaticFields.ID] == '56789' for change in changes)
self.assertTrue(status)
self.assertThat(changes, matchers.HasLength(4))
# Action
changes = self.static_physical_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
self.assertThat(changes, IsEmpty())

195
vitrage/tests/unit/datasources/static_physical/test_static_physical_transformer.py
View File

@ -1,195 +0,0 @@
# Copyright 2016 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_config import cfg
from oslo_log import log as logging
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EdgeLabel
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import UpdateMethod
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE
from vitrage.datasources.nova.host.transformer import HostTransformer
from vitrage.datasources.static import StaticFields
from vitrage.datasources.static_physical import STATIC_PHYSICAL_DATASOURCE
from vitrage.datasources.static_physical.transformer \
import StaticPhysicalTransformer
from vitrage.datasources import transformer_base as tbase
from vitrage.datasources.transformer_base import TransformerBase
from vitrage.tests import base
from vitrage.tests.mocks import mock_driver as mock_sync
LOG = logging.getLogger(__name__)
class TestStaticPhysicalTransformer(base.BaseTest):
OPTS = [
cfg.StrOpt(DSOpts.UPDATE_METHOD,
default=UpdateMethod.PULL),
]
# noinspection PyAttributeOutsideInit,PyPep8Naming
@classmethod
def setUpClass(cls):
super(TestStaticPhysicalTransformer, cls).setUpClass()
cls.transformers = {}
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group=STATIC_PHYSICAL_DATASOURCE)
cls.transformers[NOVA_HOST_DATASOURCE] = HostTransformer(
cls.transformers, cls.conf)
cls.transformers[STATIC_PHYSICAL_DATASOURCE] = \
StaticPhysicalTransformer(cls.transformers, cls.conf)
def test_create_placeholder_vertex(self):
LOG.debug('Static Physical transformer test: Create placeholder '
'vertex')
# Test setup
switch_type = STATIC_PHYSICAL_DATASOURCE
switch_name = 'switch-1'
timestamp = datetime.datetime.utcnow()
static_transformer = self.transformers[STATIC_PHYSICAL_DATASOURCE]
# Test action
properties = {
VProps.VITRAGE_TYPE: switch_type,
VProps.ID: switch_name,
VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE,
VProps.VITRAGE_SAMPLE_TIMESTAMP: timestamp
}
placeholder = \
static_transformer.create_neighbor_placeholder_vertex(**properties)
# Test assertions
observed_uuid = placeholder.vertex_id
expected_key = tbase.build_key(static_transformer._key_values(
switch_type, switch_name))
expected_uuid = \
TransformerBase.uuid_from_deprecated_vitrage_id(expected_key)
self.assertEqual(expected_uuid, observed_uuid)
observed_time = placeholder.get(VProps.VITRAGE_SAMPLE_TIMESTAMP)
self.assertEqual(timestamp, observed_time)
observed_subtype = placeholder.get(VProps.VITRAGE_TYPE)
self.assertEqual(switch_type, observed_subtype)
observed_entity_id = placeholder.get(VProps.ID)
self.assertEqual(switch_name, observed_entity_id)
observed_vitrage_category = placeholder.get(VProps.VITRAGE_CATEGORY)
self.assertEqual(EntityCategory.RESOURCE, observed_vitrage_category)
vitrage_is_placeholder = placeholder.get(VProps.VITRAGE_IS_PLACEHOLDER)
self.assertTrue(vitrage_is_placeholder)
def test_key_values(self):
LOG.debug('Static Physical transformer test: get key values')
# Test setup
switch_type = STATIC_PHYSICAL_DATASOURCE
switch_name = 'switch-1'
static_transformer = self.transformers[STATIC_PHYSICAL_DATASOURCE]
# Test action
observed_key_fields = static_transformer._key_values(switch_type,
switch_name)
# Test assertions
self.assertEqual(EntityCategory.RESOURCE, observed_key_fields[0])
self.assertEqual(STATIC_PHYSICAL_DATASOURCE, observed_key_fields[1])
self.assertEqual(switch_name, observed_key_fields[2])
def test_snapshot_transform(self):
LOG.debug('Test transform entity snapshot/snapshot_init event')
# Test setup
spec_list = mock_sync.simple_switch_generators(2, 10, 10)
static_events = mock_sync.generate_random_events_list(spec_list)
self._events_transform_test(static_events)
def test_update_transform(self):
LOG.debug('Test transform entity update event')
# Test setup
spec_list = mock_sync.simple_switch_generators(2, 10, 0, None, 10)
static_events = mock_sync.generate_random_events_list(spec_list)
self._events_transform_test(static_events)
def _events_transform_test(self, events):
for event in events:
# Test action
wrapper = self.transformers[STATIC_PHYSICAL_DATASOURCE].\
transform(event)
# Test assertions
vertex = wrapper.vertex
self._validate_switch_vertex_props(vertex, event)
neighbors = wrapper.neighbors
self._validate_neighbors(neighbors, vertex.vertex_id, event)
def _validate_neighbors(self, neighbors, switch_vertex_id, event):
host_counter = 0
for neighbor in neighbors:
self._validate_host_neighbor(neighbor,
event['relationships'][host_counter],
switch_vertex_id)
host_counter += 1
self.assertEqual(5,
host_counter,
'Zone can belongs to only one Cluster')
def _validate_host_neighbor(self,
host_neighbor,
host_event,
switch_vertex_id):
# validate neighbor vertex
self._validate_host_vertex_props(host_neighbor.vertex, host_event)
# Validate neighbor edge
edge = host_neighbor.edge
self.assertEqual(edge.target_id, switch_vertex_id)
self.assertEqual(edge.source_id, host_neighbor.vertex.vertex_id)
self.assertEqual(edge.label, EdgeLabel.CONTAINS)
def _validate_common_vertex_props(self, vertex, event):
self.assertEqual(EntityCategory.RESOURCE,
vertex[VProps.VITRAGE_CATEGORY])
self.assertEqual(event[StaticFields.TYPE],
vertex[VProps.VITRAGE_TYPE])
self.assertEqual(event[StaticFields.ID], vertex[VProps.ID])
def _validate_switch_vertex_props(self, vertex, event):
self._validate_common_vertex_props(vertex, event)
self.assertEqual(event[DSProps.SAMPLE_DATE],
vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP])
self.assertEqual(event[VProps.NAME], vertex[VProps.NAME])
self.assertEqual(event[VProps.STATE], vertex[VProps.STATE])
self.assertFalse(vertex[VProps.VITRAGE_IS_PLACEHOLDER])
self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED])
def _validate_host_vertex_props(self, vertex, event):
self._validate_common_vertex_props(vertex, event)
self.assertTrue(vertex[VProps.VITRAGE_IS_PLACEHOLDER])
self.assertFalse(vertex[VProps.VITRAGE_IS_DELETED])

5
vitrage/tests/unit/graph/base.py
View File

@ -31,7 +31,6 @@ from vitrage.common.exception import VitrageError
from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE
from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE
from vitrage.datasources import OPENSTACK_CLUSTER
from vitrage.datasources.static_physical import SWITCH
from vitrage.datasources.transformer_base import CLUSTER_ID
from vitrage.graph.driver.networkx_graph import NXGraph
from vitrage.graph import utils as graph_utils
@ -80,9 +79,9 @@ v_alarm = graph_utils.create_vertex(
metadata={VProps.RESOURCE_ID: '333333333333',
VProps.NAME: 'anotheralarm'})
v_switch = graph_utils.create_vertex(
vitrage_id=SWITCH + '1212121212',
vitrage_id='switch1212121212',
vitrage_category=RESOURCE,
vitrage_type=SWITCH,
vitrage_type='switch',
entity_id='1212121212')
e_node_to_host = graph_utils.create_edge(

6
vitrage/tests/unit/graph/test_graph_algo.py
View File

@ -111,14 +111,14 @@ class GraphAlgorithmTest(GraphTestBase):
query = {
'or': [
{'==': {VProps.VITRAGE_TYPE: SWITCH}},
{'==': {VProps.VITRAGE_TYPE: 'switch'}},
{'==': {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE}},
]
}
subgraph = ga.graph_query_vertices(
root_id=first_host_id, query_dict=query, depth=1)
self.assertEqual(
1, # For SWITCH
1, # For 'switch'
subgraph.num_edges(), 'num of BOTH edges Host (depth 1)')
subgraph = ga.graph_query_vertices(root_id=first_host_id, depth=2)
@ -266,7 +266,7 @@ class GraphAlgorithmTest(GraphTestBase):
t_v_vm_alarm = graph_utils.create_vertex(
vitrage_id='4', vitrage_category=ALARM, vitrage_type=ALARM_ON_VM)
t_v_switch = graph_utils.create_vertex(
vitrage_id='5', vitrage_category=RESOURCE, vitrage_type=SWITCH)
vitrage_id='5', vitrage_category=RESOURCE, vitrage_type='switch')
t_v_node = graph_utils.create_vertex(
vitrage_id='6',
vitrage_category=RESOURCE,