openstack
/
vitrage
Code Issues Proposed changes

Added TMF API 639 Datasource 

Change-Id: I48ccc6202738b52e80b64b965fe4993d141061f8
This commit is contained in:
Dinis Canastro 2020-04-03 19:20:54 +01:00 committed by Dinis Canastro
parent 0d9be7cb4d
commit b801bbcdbe
9 changed files with 480 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/source/contributor/configuration.rst
View File

@ -29,6 +29,7 @@ Datasources
nova-config
prometheus-datasource
kapacitor-datasource
tmfapi639-datasource
Notifiers
---------

37
doc/source/contributor/tmfapi639-datasource.rst Normal file
View File

@ -0,0 +1,37 @@
TMF API 639 - Vitrage
=====================
This datasource loads to Vitrage topologies exposed in TMF API 639 Resource Inventory Management.
https://www.tmforum.org/resources/specification/tmf639-resource-inventory-management-api-rest-specification-r17-0-1/
The fields used to define the topology will be:
- id
- name
- @type
- resourceRelationship : [resource: id]
Configuration
-------------
1. Create file ``tmfapi639_conf.yaml`` on your vitrage folder (generally: /etc/vitrage/) according to the following template:
| -endpoint:
| snapshot: URL CONTAINING COMPLETE TOPOLOGY
| update: OPTIONAL URL CONTAINING NOTIFICATIONS FOR TOPOLOGY CHANGES
You may allow as many endpoints as you desire.
2. Add tmfapi639 to list of datasources in ``/etc/vitrage/vitrage.conf``
.. code::
[datasources]
types = ...,tmfapi639,...
3. Restart vitrage service in devstack/openstack
**Warning:** due to limitations on TMF API definition, topology changes will require all parents all the way to the root to be defined in order to be correctly represented.

5
releasenotes/notes/tmfapi639_datasource-bcae9474430ae5a0.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- New TMF API 639 datasource added capable of both handling
topology snapshots and further updates. All described within
the TMF's API 639 specification.

51
vitrage/datasources/tmfapi639/__init__.py Normal file
View File

@ -0,0 +1,51 @@
# Copyright 2020
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import UpdateMethod
TMFAPI639_DATASOURCE = 'tmfapi639'
OPTS = [
cfg.StrOpt(DSOpts.TRANSFORMER,
default='vitrage.datasources.tmfapi639.transformer.'
'TmfApi639Transformer',
help='TmfApi639 transformer class path',
required=True),
cfg.StrOpt(DSOpts.DRIVER,
default='vitrage.datasources.tmfapi639.driver.'
'TmfApi639Driver',
help='TmfApi639 driver class path',
required=True),
cfg.StrOpt(DSOpts.UPDATE_METHOD,
default=UpdateMethod.PULL,
help='None: updates only via Vitrage periodic snapshots.'
'Pull: updates periodically.'
'Push: updates by getting notifications from the'
' datasource itself.',
required=True),
cfg.IntOpt(DSOpts.CHANGES_INTERVAL,
default=30,
min=10,
help='interval in seconds between checking changes in the'
' TmfApi 639 interface'),
cfg.StrOpt(DSOpts.CONFIG_FILE, default='/etc/vitrage/tmfapi639_conf.yaml',
help='TmfApi639 configuration file'
)]
class TmfApi639Fields(object):
TYPE = 'type'
ID = 'id'

51
vitrage/datasources/tmfapi639/config.py Normal file
View File

@ -0,0 +1,51 @@
# Copyright 2020
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_log import log
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.utils import file as file_utils
CONF = cfg.CONF
LOG = log.getLogger(__name__)
class TmfApi639Config(object):
def __init__(self):
try:
tmfapi639_config_file = CONF.tmfapi639[DSOpts.CONFIG_FILE]
tmfapi639_config = file_utils.load_yaml_file(tmfapi639_config_file)
self.endpoints = self._create_mapping(tmfapi639_config)
except Exception as e:
LOG.error("Failed initialization: " + str(e))
self.endpoints = []
@staticmethod
def _create_mapping(config):
"""Read URL list from config dictionary"""
LOG.debug(config)
endpoint_list = []
# Tuple list containing either 1 or 2 elements (Endpoint and updates)
for e in config:
snapshot_url = e["endpoint"]["snapshot"]
update_url = ""
if "update" in e["endpoint"]:
update_url = e["endpoint"]["update"]
if update_url != "":
endpoint_list.append((snapshot_url, update_url))
else:
endpoint_list.append(snapshot_url)
LOG.info("Finished reading endpoints file")
return endpoint_list

96
vitrage/datasources/tmfapi639/driver.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright 2020
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from vitrage.datasources.driver_base import DriverBase
from vitrage.datasources.tmfapi639 import TMFAPI639_DATASOURCE
from vitrage.datasources.tmfapi639.config import TmfApi639Config
import json
import requests
LOG = log.getLogger(__name__)
class TmfApi639Driver(DriverBase):
def __init__(self):
super(TmfApi639Driver, self).__init__()
self.config = TmfApi639Config()
self.endpoints = self.config.endpoints
self.event_lambda = 0
@staticmethod
def get_event_types():
return ['tmfapi639.instance.create',
'tmfapi639.instance.update',
'tmfapi639.instance.delete']
def enrich_event(self, event, event_type):
pass
def get_all(self, datasource_action):
"""Query all entities and send events to the vitrage events queue.
When done for the first time, send an "end" event to inform it has
finished the get_all for the datasource (because it is done
asynchronously).
"""
return self.make_pickleable(self._get_all_entities(),
TMFAPI639_DATASOURCE,
datasource_action)
def get_changes(self, datasource_action):
"""Send an event to the vitrage events queue upon any change."""
return self.make_pickleable(self._get_changes_entities(),
TMFAPI639_DATASOURCE,
datasource_action)
def _get_all_entities(self):
total = []
for pairs in self.endpoints:
try:
if type(pairs) is tuple: # Contains an update URL
LOG.info("Connecting to " + pairs[0] +
"with updates in " + pairs[1])
r = requests.get(pairs[0])
elif type(pairs) is str: # Doesn't contain update URL
LOG.info("Connecting to " + pairs)
r = requests.get(pairs)
r_dict = json.loads(r.text)
total += r_dict
except Exception as e:
LOG.error("Couldn't establish connection:" + str(e))
return total
def _get_changes_entities(self): # Called by get changes
total = []
for pairs in self.endpoints:
try:
if type(pairs) is tuple: # Contains an update URL
LOG.info("Connecting to " + pairs[0] +
"with updates in " + pairs[1])
r = requests.get(pairs[1])
r_dict = json.loads(r.text)
for e in r_dict:
if e["eventId"] < self.event_lambda:
continue
total.append(e["event"]["resource"])
self.event_lambda = e["eventId"]
except Exception as e:
LOG.error("Couldn't establish connection:" + str(e))
return total

97
vitrage/datasources/tmfapi639/transformer.py Normal file
View File

@ -0,0 +1,97 @@
# Copyright 2020
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.resource_transformer_base \
import ResourceTransformerBase
from vitrage.datasources.tmfapi639 import TMFAPI639_DATASOURCE
from vitrage.datasources import transformer_base
import vitrage.graph.utils as graph_utils
from datetime import datetime
LOG = logging.getLogger(__name__)
class TmfApi639Transformer(ResourceTransformerBase):
def __init__(self, transformers):
super(TmfApi639Transformer, self).__init__(transformers)
def _create_snapshot_entity_vertex(self, entity_event):
return self._create_vertex(entity_event)
def _create_update_entity_vertex(self, entity_event):
return self._create_vertex(entity_event)
def _create_snapshot_neighbors(self, entity_event):
return self._create_tmfapi639_neighbors(entity_event)
def _create_update_neighbors(self, entity_event):
return self._create_tmfapi639_neighbors(entity_event)
def _create_entity_key(self, entity_event):
"""the unique key of this entity"""
entity_id = entity_event["id"]
entity_type = TMFAPI639_DATASOURCE
key_fields = self._key_values(entity_type, entity_id)
return transformer_base.build_key(key_fields)
@staticmethod
def get_vitrage_type():
return TMFAPI639_DATASOURCE
def _create_vertex(self, entity_event):
"""Camps used from the received JSON:
{id, name, @type ,resourceRelationship : [type, resource: id]}
The TMF 639 API REST Endpoint can contain more information
but we only use this one for topology.
"""
sample_timestamp = \
datetime.now().strftime(transformer_base.TIMESTAMP_FORMAT)
update_timestamp = self._format_update_timestamp(
update_timestamp=None,
sample_timestamp=sample_timestamp)
metadata = {
VProps.NAME: entity_event["name"],
}
return graph_utils.create_vertex(
self._create_entity_key(entity_event),
vitrage_category=EntityCategory.RESOURCE,
vitrage_type=TMFAPI639_DATASOURCE,
vitrage_sample_timestamp=sample_timestamp,
entity_id=entity_event["id"],
update_timestamp=update_timestamp,
entity_state='available',
metadata=metadata)
def _create_tmfapi639_neighbors(self, entity_event):
neighbors_list = []
for n in entity_event["resourceRelationship"]:
# create placeholder vertex
neigh = self._create_neighbor(
entity_event,
n["resource"]["id"],
TMFAPI639_DATASOURCE,
n["type"],
is_entity_source=True)
neighbors_list.append(neigh)
return neighbors_list

0
vitrage/tests/unit/datasources/tmfapi639/__init__.py Normal file
View File

142
vitrage/tests/unit/datasources/tmfapi639/test_tmfapi639_transformer.py Normal file
View File

@ -0,0 +1,142 @@
# Copyright 2020
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_log import log as logging
from testtools import matchers
from vitrage.common.constants import DatasourceAction
from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import UpdateMethod
from vitrage.datasources.tmfapi639 import TMFAPI639_DATASOURCE
from vitrage.datasources.tmfapi639.transformer import TmfApi639Transformer
from vitrage.datasources import transformer_base
from vitrage.datasources.transformer_base import TransformerBase
from vitrage.tests.unit.datasources.test_alarm_transformer_base import \
BaseAlarmTransformerTest
from datetime import datetime
from json import loads
LOG = logging.getLogger(__name__)
message = '[{"id":"1","name":"Host-1","@type":"Host",\
"resourceRelationship":[{"type":"parent","resource":{"id":"1"}}]},\
{"id":"2","name":"Host-2","@type":"Host",\
"resourceRelationship":[{"type":"parent","resource":{"id":"1"}}]}]'
# noinspection PyProtectedMember
class TestTmfApi639Transformer(BaseAlarmTransformerTest):
OPTS = [
cfg.StrOpt(DSOpts.UPDATE_METHOD,
default=UpdateMethod.PULL),
]
# noinspection PyAttributeOutsideInit,PyPep8Naming
@classmethod
def setUpClass(cls):
super(TestTmfApi639Transformer, cls).setUpClass()
cls.transformers = {}
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group=TMFAPI639_DATASOURCE)
cls.transformer = TmfApi639Transformer(cls.transformers)
cls.transformers[TMFAPI639_DATASOURCE] = cls.transformer
# noinspection PyAttributeOutsideInit
def setUp(self):
super(TestTmfApi639Transformer, self).setUp()
# self.entity_type = TMFAPI639_DATASOURCE
# self.entity_id = '12345'
self.timestamp = datetime.utcnow()
def test_create_entity_key(self):
event = loads(message)[0]
self.assertIsNotNone(event)
transformer = TmfApi639Transformer(self.transformers)
observed_key = transformer._create_entity_key(event)
entity_type = TMFAPI639_DATASOURCE
entity_id = event["id"]
# Test assertions
observed_key_fields = observed_key.split(
TransformerBase.KEY_SEPARATOR)
self.assertEqual(entity_type, observed_key_fields[1])
self.assertEqual(entity_id, observed_key_fields[2])
# Transformer tests:
# - Vertex creation
# - Neighbor link
def test_topology(self):
sample_timestamp = \
datetime.now().strftime(transformer_base.TIMESTAMP_FORMAT)
update_timestamp = TransformerBase._format_update_timestamp(
update_timestamp=None,
sample_timestamp=sample_timestamp)
transformer = self.transformers[TMFAPI639_DATASOURCE]
# Create 1 vertex
event1 = loads(message)[0]
event1[DSProps.DATASOURCE_ACTION] = DatasourceAction.SNAPSHOT
event1[DSProps.SAMPLE_DATE] = update_timestamp
self.assertIsNotNone(event1)
# Create vertex 1
wrapper1 = transformer.transform(event1)
# Assertion
self._validate_base_vertex_props(
wrapper1.vertex,
event1["name"],
TMFAPI639_DATASOURCE
)
# Create 2nd vertex
event2 = loads(message)[1]
event2[DSProps.DATASOURCE_ACTION] = DatasourceAction.SNAPSHOT
event2[DSProps.SAMPLE_DATE] = update_timestamp
self.assertIsNotNone(event2)
# Create vertex 2
wrapper2 = transformer.transform(event2)
# Assertion
self._validate_base_vertex_props(
wrapper2.vertex,
event2["name"],
TMFAPI639_DATASOURCE
)
# Test whether they are linked
self.assertThat(wrapper2.neighbors, matchers.HasLength(1))
parent_id = transformer._create_entity_key(event1)
parent_uuid = \
transformer.uuid_from_deprecated_vitrage_id(parent_id)
child_id = transformer._create_entity_key(event2)
child_uuid = \
transformer.uuid_from_deprecated_vitrage_id(child_id)
self.assertEqual(wrapper2.neighbors[0].edge.source_id, child_uuid)
self.assertEqual(wrapper2.neighbors[0].edge.target_id, parent_uuid)