openstack
/
vitrage
Code Issues Proposed changes

Merge "Support get_changes in the static datasource"

This commit is contained in:
Zuul 2018-06-07 08:00:03 +00:00 committed by Gerrit Code Review
parent 5dd550a0c0 d5b4daf8aa
commit 28716bc2d9
10 changed files with 369 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
releasenotes/notes/static-datasource-changes-914f9a16ad7e46ed.yaml Normal file
View File

@ -0,0 +1,3 @@
---
features:
- Support get_changes in the static datasource

4
releasenotes/notes/support-get-changes-in-static-datasource-02715226f103455d.yaml Normal file
View File

@ -0,0 +1,4 @@
---
features:
- The static datasource now supports changes in existing yaml files, and
updates the graph accordingly.

1
vitrage/datasources/static/__init__.py
View File

@ -62,3 +62,4 @@ class StaticFields(object):
CATEGORY = 'category'
ID = 'id'
NAME = 'name'
STATE = 'state'

63
vitrage/datasources/static/driver.py
View File

@ -17,6 +17,8 @@ from six.moves import reduce
from oslo_log import log
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import GraphAction
from vitrage.datasources.driver_base import DriverBase
from vitrage.datasources.static import STATIC_DATASOURCE
from vitrage.datasources.static import StaticFields
@ -34,6 +36,7 @@ class StaticDriver(DriverBase):
def __init__(self, conf):
super(StaticDriver, self).__init__()
self.cfg = conf
self.entities_cache = []
@staticmethod
def _is_valid_config(config):
@ -49,23 +52,48 @@ class StaticDriver(DriverBase):
pass
def get_all(self, datasource_action):
return self.make_pickleable(self._get_all_entities(),
return self.make_pickleable(self._get_and_cache_all_entities(),
STATIC_DATASOURCE,
datasource_action)
def get_changes(self, datasource_action):
return self.make_pickleable(self._get_changes_entities(),
return self.make_pickleable(self._get_and_cache_changed_entities(),
STATIC_DATASOURCE,
datasource_action)
def _get_and_cache_all_entities(self):
self.entities_cache = self._get_all_entities()
return self.entities_cache
def _get_all_entities(self):
files = file_utils.list_files(self.cfg.static.directory, '.yaml', True)
return reduce(chain, [self._get_entities_from_file(path)
for path in files], [])
return list(reduce(chain, [self._get_entities_from_file(path)
for path in files], []))
def _get_changes_entities(self):
"""TODO(yujunz): update from file change or CRUD"""
return []
def _get_and_cache_changed_entities(self):
changed_entities = []
new_entities = self._get_all_entities()
for new_entity in new_entities:
old_entity = self._find_entity(new_entity, self.entities_cache)
if old_entity:
# Add modified entities
if not self._equal_entities(old_entity, new_entity):
changed_entities.append(new_entity.copy())
else:
# Add new entities
changed_entities.append(new_entity.copy())
# Add deleted entities
for old_entity in self.entities_cache:
if not self._find_entity(old_entity, new_entities):
old_entity_copy = old_entity.copy()
old_entity_copy[DSProps.EVENT_TYPE] = GraphAction.DELETE_ENTITY
changed_entities.append(old_entity_copy)
self.entities_cache = new_entities
return changed_entities
@classmethod
def _get_entities_from_file(cls, path):
@ -140,3 +168,24 @@ class StaticDriver(DriverBase):
.format(neighbor, rel))
return None
return rel
@staticmethod
def _find_entity(search_entity, entities):
# naive implementation since we don't expect many static entities
for entity in entities:
if entity[StaticFields.TYPE] == search_entity[StaticFields.TYPE] \
and entity[StaticFields.ID] == \
search_entity[StaticFields.ID]:
return entity
@staticmethod
def _equal_entities(old_entity, new_entity):
# TODO(iafek): compare also the relationships
return old_entity.get(StaticFields.TYPE) == \
new_entity.get(StaticFields.TYPE) and \
old_entity.get(StaticFields.ID) == \
new_entity.get(StaticFields.ID) and \
old_entity.get(StaticFields.NAME) == \
new_entity.get(StaticFields.NAME) and \
old_entity.get(StaticFields.STATE) == \
new_entity.get(StaticFields.STATE)

48
vitrage/tests/resources/static_datasources/changes_datasources/added_resources/static.yaml Normal file
View File

@ -0,0 +1,48 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s1
type: switch
name: switch-1
id: 12345
state: available
- static_id: s2
type: switch
name: switch-2
id: 23456
state: available
- static_id: s3
type: switch
name: switch-3 is new!
id: 3333
state: available
- static_id: r1
type: router
name: router-1
id: 45678
- static_id: r2
type: router
name: router-2 is new!
id: 2222
state: available
- static_id: h1
type: nova.host
id: 1
relationships:
- source: s1
target: r1
relationship_type: attached
- source: s2
target: r1
relationship_type: attached
- source: s3
target: r2
relationship_type: attached
- source: r1
target: h1
relationship_type: attached
- source: r2
target: h1
relationship_type: attached

32
vitrage/tests/resources/static_datasources/changes_datasources/baseline/static.yaml Normal file
View File

@ -0,0 +1,32 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s1
type: switch
name: switch-1
id: 12345
state: available
- static_id: s2
type: switch
name: switch-2
id: 23456
state: available
- static_id: r1
type: router
name: router-1
id: 45678
- static_id: h1
type: nova.host
id: 1
relationships:
- source: s1
target: r1
relationship_type: attached
- source: s2
target: r1
relationship_type: attached
- source: r1
target: h1
relationship_type: attached

32
vitrage/tests/resources/static_datasources/changes_datasources/changed_resources/static.yaml Normal file
View File

@ -0,0 +1,32 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s1
type: switch
name: switch-1
id: 12345
state: error # state change
- static_id: s2
type: switch
name: switch-2
id: 23456
state: available
- static_id: r1
type: router
name: router-1 is the best! # name change
id: 45678
- static_id: h1
type: nova.host
id: 1
relationships:
- source: s1
target: r1
relationship_type: attached
- source: s2
target: r1
relationship_type: attached
- source: r1
target: h1
relationship_type: attached

18
vitrage/tests/resources/static_datasources/changes_datasources/deleted_resources/static.yaml Normal file
View File

@ -0,0 +1,18 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s1
type: switch
name: switch-1
id: 12345
state: available
- static_id: r1
type: router
name: router-1
id: 45678
relationships:
- source: s1
target: r1
relationship_type: attached

31
vitrage/tests/resources/static_datasources/changes_datasources/mixed_changes/static.yaml Normal file
View File

@ -0,0 +1,31 @@
metadata:
name: switch to host
description: static datasource for test
definitions:
entities:
- static_id: s2
type: switch
name: switch-2
id: 23456
state: error
- static_id: r1
type: router
name: router-1
id: 45678
- static_id: r2
type: router
name: router-2 is new!
id: 222
- static_id: h1
type: nova.host
id: 1
relationships:
- source: s2
target: r1
relationship_type: attached
- source: r1
target: h1
relationship_type: attached
- source: r2
target: h1
relationship_type: attached

191
vitrage/tests/unit/datasources/static/test_static_driver.py
View File

@ -22,54 +22,22 @@ from vitrage.datasources.static import driver
from vitrage.datasources.static import STATIC_DATASOURCE
from vitrage.datasources.static import StaticFields
from vitrage.tests import base
from vitrage.tests.base import IsEmpty
from vitrage.tests.mocks import utils
class TestStaticDriver(base.BaseTest):
OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.static.transformer.'
'StaticTransformer'),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.static.driver.'
'StaticDriver'),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=30,
help='interval between checking changes in the '
'configuration files of the static datasources'),
cfg.StrOpt('directory',
default=utils.get_resources_dir() + '/static_datasources')
]
CHANGES_OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.static.transformer.'
'StaticTransformer'),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.static.driver.'
'StaticDriver'),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=30,
help='interval between checking changes in the static '
'datasources'),
cfg.StrOpt('directory',
default=utils.get_resources_dir() +
'/static_datasources/changes_datasources'),
]
CHANGES_DIR = '/changes_datasources'
# noinspection PyAttributeOutsideInit,PyPep8Naming
@classmethod
def setUpClass(cls):
super(TestStaticDriver, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group=STATIC_DATASOURCE)
cls.static_driver = driver.StaticDriver(cls.conf)
cls.static_driver = driver.StaticDriver(None)
def test_get_all(self):
self._set_conf()
# Action
static_entities = self.static_driver.get_all(
DatasourceAction.INIT_SNAPSHOT)
@ -81,24 +49,108 @@ class TestStaticDriver(base.BaseTest):
self._validate_static_entity(entity)
# noinspection PyAttributeOutsideInit
def test_get_changes(self):
# Setup
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(8))
def test_get_changes_with_added_resources(self):
# Get initial resources
self._set_conf(self.CHANGES_DIR + '/baseline')
self.conf = cfg.ConfigOpts()
self.conf.register_opts(self.CHANGES_OPTS,
group=STATIC_DATASOURCE)
self.static_driver.cfg = self.conf
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(4))
# Add resources
self._set_conf(self.CHANGES_DIR + '/added_resources')
# Action
changes = self.static_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
self.assertThat(changes, IsEmpty())
for entity in changes:
self._validate_static_entity(entity)
expected_changes = [
{'static_id': 's3', 'type': 'switch', 'name': 'switch-3 is new!',
'id': '3333', 'state': 'available'},
{'static_id': 'r2', 'type': 'router', 'name': 'router-2 is new!',
'id': '2222', 'state': 'available'},
]
self._validate_static_changes(expected_changes, changes)
# noinspection PyAttributeOutsideInit
def test_get_changes_with_deleted_resources(self):
# Get initial resources
self._set_conf(self.CHANGES_DIR + '/baseline')
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(4))
# Delete resources
self._set_conf(self.CHANGES_DIR + '/deleted_resources')
# Action
changes = self.static_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
expected_changes = [
{'static_id': 's2', 'type': 'switch', 'name': 'switch-2',
'id': '23456', 'state': 'available',
'vitrage_event_type': 'delete_entity'},
{'static_id': 'h1', 'type': 'nova.host', 'id': '1',
'vitrage_event_type': 'delete_entity'},
]
self._validate_static_changes(expected_changes, changes)
# noinspection PyAttributeOutsideInit
def test_get_changes_with_changed_resources(self):
# Get initial resources
self._set_conf(self.CHANGES_DIR + '/baseline')
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(4))
# Delete resources
self._set_conf(self.CHANGES_DIR + '/changed_resources')
# Action
changes = self.static_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
expected_changes = [
{'static_id': 's1', 'type': 'switch', 'name': 'switch-1',
'id': '12345', 'state': 'error'},
{'static_id': 'r1', 'type': 'router',
'name': 'router-1 is the best!', 'id': '45678'},
]
self._validate_static_changes(expected_changes, changes)
# noinspection PyAttributeOutsideInit
def test_get_changes_with_mixed_changes(self):
# Get initial resources
self._set_conf(self.CHANGES_DIR + '/baseline')
entities = self.static_driver.get_all(DatasourceAction.UPDATE)
self.assertThat(entities, matchers.HasLength(4))
# Delete resources
self._set_conf(self.CHANGES_DIR + '/mixed_changes')
# Action
changes = self.static_driver.get_changes(
GraphAction.UPDATE_ENTITY)
# Test Assertions
expected_changes = [
{'static_id': 's1', 'type': 'switch', 'name': 'switch-1',
'id': '12345', 'state': 'available',
'vitrage_event_type': 'delete_entity'},
{'static_id': 's2', 'type': 'switch', 'name': 'switch-2',
'id': '23456', 'state': 'error'},
{'static_id': 'r2', 'type': 'router', 'name': 'router-2 is new!',
'id': '222'},
]
self._validate_static_changes(expected_changes, changes)
def _validate_static_entity(self, entity):
self.assertIsInstance(entity[StaticFields.METADATA], dict)
@ -115,3 +167,48 @@ class TestStaticDriver(base.BaseTest):
and entity[StaticFields.STATIC_ID] == rel[StaticFields.SOURCE]
or entity[StaticFields.STATIC_ID] == rel[StaticFields.SOURCE]
and entity[StaticFields.STATIC_ID] == rel[StaticFields.TARGET])
def _validate_static_changes(self, expected_changes, changes):
self.assertThat(changes, matchers.HasLength(len(expected_changes)))
for entity in changes:
self._validate_static_entity(entity)
for expected_change in expected_changes:
found = False
for change in changes:
if change[StaticFields.TYPE] == \
expected_change[StaticFields.TYPE] and \
change[StaticFields.ID] == \
expected_change[StaticFields.ID]:
found = True
self.assertEqual(expected_change.get('vitrage_event_type'),
change.get('vitrage_event_type'))
self.assertEqual(expected_change.get(StaticFields.NAME),
change.get(StaticFields.NAME))
self.assertEqual(expected_change.get(StaticFields.STATE),
change.get(StaticFields.STATE))
self.assertTrue(found)
def _set_conf(self, sub_dir=None):
default_dir = utils.get_resources_dir() + \
'/static_datasources' + (sub_dir if sub_dir else '')
opts = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.static.transformer.'
'StaticTransformer'),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.static.driver.'
'StaticDriver'),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=30,
help='interval between checking changes in the static '
'datasources'),
cfg.StrOpt('directory', default=default_dir),
]
self.conf = cfg.ConfigOpts()
self.conf.register_opts(opts, group=STATIC_DATASOURCE)
self.static_driver.cfg = self.conf