openstack
/
congress
Code Issues Proposed changes

Adding Monasca Driver to Datasources 

Monasca is the Monitoring service for OpenStack. This patch adds
it to the list of Datasources.

Monasca supports Metrics, Measurements, Alarms and Statistics.
API calls for Metrics, Statistics and Alarms have been added
and the data represented as tables for the Congress data collection.

Measurements have been on purpose left out because it can really
flood the system.

In the future we want to have a push model integration rathen than
a poll model as implemented here.

Implements: blueprint add-monasca-datasource-driver

Change-Id: Ib8db6ba8d075798641eb21e937bc3f28135a29c0
This commit is contained in:
Fabio Giannetti 2015-11-04 16:14:43 -08:00
parent 144cbacb98
commit 293528aab5
6 changed files with 319 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
congress/datasources/datasource_utils.py
View File

@ -30,6 +30,7 @@ def get_openstack_required_config():
'username': constants.REQUIRED,
'password': constants.REQUIRED,
'tenant_name': constants.REQUIRED,
'project_name': constants.OPTIONAL,
'poll_time': constants.OPTIONAL}

184
congress/datasources/monasca_driver.py Normal file
View File

@ -0,0 +1,184 @@
# Copyright (c) 2015 Cisco.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import datetime
import keystoneclient.v3.client as ksclient
from monascaclient import client as monasca_client
from oslo_log import log as logging
from oslo_utils import timeutils
from congress.datasources import datasource_driver
from congress.datasources import datasource_utils as ds_utils
LOG = logging.getLogger(__name__)
def d6service(name, keys, inbox, datapath, args):
"""Create a dataservice instance.
This method is called by d6cage to create a dataservice
instance. There are a couple of parameters we found useful
to add to that call, so we included them here instead of
modifying d6cage (and all the d6cage.createservice calls).
"""
return MonascaDriver(name, keys, inbox, datapath, args)
# TODO(thinrichs): figure out how to move even more of this boilerplate
# into DataSourceDriver. E.g. change all the classes to Driver instead of
# NeutronDriver, CeilometerDriver, etc. and move the d6instantiate function
# to DataSourceDriver.
class MonascaDriver(datasource_driver.PollingDataSourceDriver,
datasource_driver.ExecutionDriver):
METRICS = "metrics"
DIMENSIONS = "dimensions"
STATISTICS = "statistics"
DATA = "statistics.data"
# TODO(fabiog): add events and logs when fully supported in Monasca
# EVENTS = "events"
# LOGS = "logs"
value_trans = {'translation-type': 'VALUE'}
metric_translator = {
'translation-type': 'HDICT',
'table-name': METRICS,
'selector-type': 'DICT_SELECTOR',
'field-translators':
({'fieldname': 'id', 'translator': value_trans},
{'fieldname': 'name', 'translator': value_trans},
{'fieldname': 'dimensions',
'translator': {'translation-type': 'VDICT',
'table-name': DIMENSIONS,
'id-col': 'id',
'key-col': 'key', 'val-col': 'value',
'translator': value_trans}})
}
statistics_translator = {
'translation-type': 'HDICT',
'table-name': STATISTICS,
'selector-type': 'DICT_SELECTOR',
'field-translators':
({'fieldname': 'name', 'translator': value_trans},
{'fieldname': 'statistics',
'translator': {'translation-type': 'LIST',
'table-name': DATA,
'id-col': 'name',
'val-col': 'value_col',
'translator': value_trans}})
}
TRANSLATORS = [metric_translator, statistics_translator]
def __init__(self, name='', keys='', inbox=None, datapath=None,
args=None):
super(MonascaDriver, self).__init__(name, keys, inbox, datapath, args)
datasource_driver.ExecutionDriver.__init__(self)
self.creds = args
if not self.creds.get('project_name'):
self.creds['project_name'] = self.creds['tenant_name']
if not self.creds.get('poll_time'):
# set default polling time to 1hr
self.creds['poll_time'] = 3600
# Monasca uses Keystone V3
self.creds['auth_url'] = self.creds['auth_url'].replace("v2.0", "v3")
self.keystone = ksclient.Client(**self.creds)
self.creds['token'] = self.keystone.auth_token
if not self.creds.get('endpoint'):
# if the endpoint not defined retrieved it from keystone catalog
self.creds['endpoint'] = self.keystone.service_catalog.url_for(
service_type='monitoring', endpoint_type='publicURL')
self.monasca = monasca_client.Client('2_0', **self.creds)
self.add_executable_client_methods(self.monasca, 'monascaclient.')
self._init_end_start_poll()
@staticmethod
def get_datasource_info():
result = {}
result['id'] = 'monasca'
result['description'] = ('Datasource driver that interfaces with '
'monasca.')
result['config'] = ds_utils.get_openstack_required_config()
result['secret'] = ['password']
return result
def update_from_datasource(self):
"""Read Data from Monasca datasource.
And to fill up the current state of the policy engine.
"""
try:
LOG.debug("Monasca grabbing metrics")
metrics = self.monasca.metrics.list()
self._translate_metric(metrics)
LOG.debug("METRICS: %s" % str(self.state[self.METRICS]))
LOG.debug("Monasca grabbing statistics")
# gather statistic for the last day
today = datetime.datetime.now()
yesterday = datetime.timedelta(hours=24)
start_from = timeutils.isotime(today-yesterday)
for metric in self.monasca.metrics.list_names():
LOG.debug("Monasca statistics for metric %s", metric['name'])
_query_args = dict(
start_time=start_from,
name=metric['name'],
statistics='avg',
period=int(self.creds['poll_time']),
merge_metrics='true')
statistics = self.monasca.metrics.list_statistics(
**_query_args)
self._translate_statistics(statistics)
LOG.debug("STATISTICS: %s" % str(self.state[self.STATISTICS]))
except Exception as e:
raise e
@ds_utils.update_state_on_changed(METRICS)
def _translate_metric(self, obj):
"""Translate the metrics represented by OBJ into tables."""
LOG.debug("METRIC: %s" % str(obj))
row_data = MonascaDriver.convert_objs(obj,
self.metric_translator)
return row_data
@ds_utils.update_state_on_changed(STATISTICS)
def _translate_statistics(self, obj):
"""Translate the metrics represented by OBJ into tables."""
LOG.debug("STATISTICS: %s" % str(obj))
row_data = MonascaDriver.convert_objs(obj,
self.statistics_translator)
return row_data
def execute(self, action, action_args):
"""Overwrite ExecutionDriver.execute()."""
# action can be written as a method or an API call.
func = getattr(self, action, None)
if func and self.is_executable(func):
func(action_args)
else:
self._execute_api(self.monasca, action, action_args)

1
congress/tests/api/test_driver_model.py
View File

@ -76,6 +76,7 @@ class TestDriverModel(base.SqlTestCase):
"password": "required",
"poll_time": "(optional)",
"region": "(optional)",
"project_name": "(optional)",
"tenant_name": "required",
"username": "required"
},

131
congress/tests/datasources/test_monasca_driver.py Normal file
View File

@ -0,0 +1,131 @@
# Copyright (c) 2015 Cisco, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import mock
import sys
sys.modules['monascaclient.client'] = mock.Mock()
sys.modules['monascaclient'] = mock.Mock()
from congress.datasources import monasca_driver
from congress.tests import base
from congress.tests import helper
class TestMonascaDriver(base.TestCase):
def setUp(self):
super(TestMonascaDriver, self).setUp()
self.keystone_client_p = mock.patch(
"keystoneclient.v3.client.Client")
self.keystone_client_p.start()
self.monasca_client_p = mock.patch("monascaclient.client.Client")
self.monasca_client_p.start()
args = helper.datasource_openstack_args()
args['poll_time'] = 0
args['client'] = mock.MagicMock()
self.driver = monasca_driver.MonascaDriver(args=args)
self.mock_metrics = {"links": [{
"rel": "self",
"href": "http://192.168.10.4:8070/v2.0/metrics"}],
"elements": [{
"id": "0",
"name": "mem.used_buffers",
"dimensions": {
"component": "monasca-persister",
"service": "monitoring",
"hostname": "ceilosca",
"url": "http://localhost:8091/metrics"}
}]}
self.mock_statistics = {"links": [{
"rel": "self",
"href": "http://192.168.10.4:8070/v2.0/metrics/statistics"}],
"elements": [{
"id": "2015-11-30T18:00:00Z",
"name": "mem.used_buffers",
"dimensions": {},
"columns": ["timestamp", "avg"],
"statistics": [
["2015-11-24T00:00:00Z", 56],
["2015-11-24T06:00:00Z", 46],
["2015-11-24T12:00:00Z", 70],
["2015-11-24T18:00:00Z", 60]]
}]}
def test_statistics_update_from_datasource(self):
self.driver._translate_statistics(self.mock_statistics['elements'])
stats_list = list(self.driver.state[self.driver.STATISTICS])
stats_data_list = list(self.driver.state[self.driver.DATA])
self.assertIsNotNone(stats_list)
self.assertIsNotNone(stats_data_list)
expected_stats = [
('mem.used_buffers', 'd1fea02438d17fb7446255573bf54d45')]
self.assertEqual(expected_stats, stats_list)
expected_stats_data = [
('d1fea02438d17fb7446255573bf54d45',
"['2015-11-24T00:00:00Z', 56]"),
('d1fea02438d17fb7446255573bf54d45',
"['2015-11-24T12:00:00Z', 70]"),
('d1fea02438d17fb7446255573bf54d45',
"['2015-11-24T18:00:00Z', 60]"),
('d1fea02438d17fb7446255573bf54d45',
"['2015-11-24T06:00:00Z', 46]")]
self.assertEqual(sorted(expected_stats_data), sorted(stats_data_list))
def test_metrics_update_from_datasource(self):
with mock.patch.object(self.driver.monasca.metrics, "list") as metrics:
metrics.return_value = self.mock_metrics['elements']
self.driver.update_from_datasource()
expected = {
'dimensions': set([
('e138b5d90a4265c7525f480dd988210b',
'component', 'monasca-persister'),
('e138b5d90a4265c7525f480dd988210b',
'service', 'monitoring'),
('e138b5d90a4265c7525f480dd988210b',
'hostname', 'ceilosca'),
('e138b5d90a4265c7525f480dd988210b',
'url', 'http://localhost:8091/metrics')]),
'metrics': set([
('0', 'mem.used_buffers',
'e138b5d90a4265c7525f480dd988210b')]),
'statistics': set([]),
'statistics.data': set([])
}
self.assertEqual(expected, self.driver.state)
def test_execute(self):
class MonascaClient(object):
def __init__(self):
self.testkey = None
def getStatistics(self, arg1):
self.testkey = 'arg1=%s' % arg1
monasca_client = MonascaClient()
self.driver.monasca = monasca_client
api_args = {
'positional': ['1']
}
expected_ans = 'arg1=1'
self.driver.execute('getStatistics', api_args)
self.assertEqual(expected_ans, monasca_client.testkey)

1
requirements.txt
View File

@ -1,6 +1,7 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
Babel>=1.3 # BSD
eventlet!=0.18.0,>=0.17.4 # MIT
PuLP>=1.0.4 # MIT

1
thirdparty-requirements.txt
View File

@ -1,3 +1,4 @@
# This file contains a list of optional requirements that are not in the
# offical global-requirements.txt
python-cloudfoundryclient>=1.0.2
python-monascaclient>=1.0.27 # Apache-2.0