openstack/watcher
Code Issues Proposed changes

Merge "Remove monasca datasource"

Browse Source
This commit is contained in:
Zuul
2026-02-19 21:27:22 +00:00
committed by Gerrit Code Review
parent 9d5a584c9b 6653953fb5
commit 93a63e28ee
19 changed files with 33 additions and 753 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
doc/source/contributor/plugin/strategy-plugin.rst
View File

@@ -242,9 +242,9 @@ Querying metrics
A large set of metrics, generated by OpenStack modules, can be used in your
strategy implementation. To collect these metrics, Watcher provides a
`DataSourceManager`_ for two data sources which are `Ceilometer`_
(with `Gnocchi`_ as API) and `Monasca`_. If you wish to query metrics from a
different data source, you can implement your own and use it via
`DataSourceManager`_ for several data sources including `Gnocchi`_ and
`Aetos`_. If you wish to query metrics from a different data source,
you can implement your own and use it via
DataSourceManager from within your new strategy. Indeed, strategies in Watcher
have the cluster data models decoupled from the data sources which means that
you may keep the former while changing the latter. The recommended way for you
@@ -264,7 +264,7 @@ requires new metrics not covered by Ceilometer, you can add them through a
.. _`DataSourceManager`: https://github.com/openstack/watcher/blob/master/watcher/datasource/manager.py
.. _`Ceilometer developer guide`: https://docs.openstack.org/ceilometer/latest/contributor/architecture.html#storing-accessing-the-data
.. _`Ceilometer`: https://docs.openstack.org/ceilometer/latest
.. _`Monasca`: https://github.com/openstack/monasca-api/blob/master/docs/monasca-api-spec.md
.. _`Aetos`: https://docs.openstack.org/aetos/latest/
.. _`here`: https://docs.openstack.org/ceilometer/latest/contributor/install/dbreco.html#choosing-a-database-backend
.. _`Ceilometer plugin`: https://docs.openstack.org/ceilometer/latest/contributor/plugins.html
.. _`Ceilosca`: https://github.com/openstack/monasca-ceilometer/blob/master/ceilosca/ceilometer/storage/impl_monasca.py

5
doc/source/datasources/index.rst
View File

@@ -1,11 +1,6 @@
Datasources
===========
.. note::
The Monasca datasource is deprecated for removal and optional. To use it, install the optional extra:
``pip install watcher[monasca]``. If Monasca is configured without installing the extra, Watcher will raise
an error guiding you to install the client.
.. toctree::
:glob:
:maxdepth: 1

2
doc/source/images/architecture.svg
View File

@@ -627,7 +627,7 @@
x="-100.86156"
y="268.36258"
style="font-size:11.2512598px;text-align:center;text-anchor:middle"
id="tspan5184-3-5-2-1-8">monasca</tspan></text>
id="tspan5184-3-5-2-1-8">aetos</tspan></text>
</g>
<g
transform="matrix(1.7775787,0,0,1.7775787,805.08607,598.49372)"
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 64 KiB

After

Width:  |  Height:  |  Size: 64 KiB

21
doc/source/strategies/strategy-template.rst
View File

@@ -24,22 +24,17 @@ Write here the list of metrics required by your strategy algorithm (in the form
Example:
======================= ============ ======= =======
metric service name plugins comment
======================= ============ ======= =======
compute.node.* ceilometer_ none one point every 60s
vm.cpu.utilization_perc monasca_ none
power ceilometer_ kwapi_ one point every 60s
======================= ============ ======= =======
======================= ============== ======= =======
metric service name plugins comment
======================= ============== ======= =======
compute.node.* ceilometer_ none one point every 60s
instance_cpu_usage gnocchi_ none
power ceilometer_ kwapi_ one point every 60s
======================= ============== ======= =======
.. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
.. _monasca: https://github.com/openstack/monasca-agent/blob/master/docs/Libvirt.md
.. note::
The Monasca datasource is deprecated for removal and optional. If a strategy requires Monasca metrics,
ensure the Monasca optional extra is installed: ``pip install watcher[monasca]``.
.. _gnocchi: https://docs.openstack.org/gnocchi/latest/
.. _kwapi: https://kwapi.readthedocs.io/en/latest/index.html

7
releasenotes/notes/remove-monasca-datasource-e70fec5b837d44.yaml Normal file
View File

@@ -0,0 +1,7 @@
---
upgrade:
- |
The Monasca datasource has been removed from Watcher. This datasource
was deprecated in a previous release due to upstream Monasca project
retirement. Users should migrate to Gnocchi or Aetos
datasources, which provide equivalent and enhanced functionality.

4
setup.cfg
View File

@@ -27,10 +27,6 @@ packages =
data_files =
etc/ = etc/*
[options.extras_require]
monasca =
python-monascaclient>=1.12.0
[entry_points]
oslo.config.opts =
watcher = watcher.conf.opts:list_opts

46
watcher/common/clients.py
View File

@@ -70,7 +70,6 @@ class OpenStackClients:
self._nova = None
self._gnocchi = None
self._cinder = None
self._monasca = None
self._ironic = None
self._maas = None
self._placement = None
@@ -164,51 +163,6 @@ class OpenStackClients:
session=self.session)
return self._cinder
@exception.wrap_keystone_exception
def monasca(self):
if self._monasca:
return self._monasca
try:
from monascaclient import client as monclient
except ImportError as e:
message = (
"Monasca client is not installed. "
"Install 'watcher[monasca]' or "
"add 'python-monascaclient' to your environment."
)
raise exception.UnsupportedError(message) from e
monascaclient_version = self._get_client_option(
'monasca', 'api_version')
monascaclient_interface = self._get_client_option(
'monasca', 'interface')
monascaclient_region = self._get_client_option(
'monasca', 'region_name')
token = self.session.get_token()
watcher_clients_auth_config = CONF.get(_CLIENTS_AUTH_GROUP)
service_type = 'monitoring'
monasca_kwargs = {
'auth_url': watcher_clients_auth_config.auth_url,
'cert_file': watcher_clients_auth_config.certfile,
'insecure': watcher_clients_auth_config.insecure,
'key_file': watcher_clients_auth_config.keyfile,
'keystone_timeout': watcher_clients_auth_config.timeout,
'os_cacert': watcher_clients_auth_config.cafile,
'service_type': service_type,
'token': token,
'username': watcher_clients_auth_config.username,
'password': watcher_clients_auth_config.password,
}
endpoint = self.session.get_endpoint(service_type=service_type,
interface=monascaclient_interface,
region_name=monascaclient_region)
self._monasca = monclient.Client(
monascaclient_version, endpoint, **monasca_kwargs)
return self._monasca
@exception.wrap_keystone_exception
def ironic(self):
if self._ironic:

2
watcher/conf/__init__.py
View File

@@ -35,7 +35,6 @@ from watcher.conf import ironic_client
from watcher.conf import keystone_client
from watcher.conf import maas_client
from watcher.conf import models
from watcher.conf import monasca_client
from watcher.conf import nova
from watcher.conf import nova_client
from watcher.conf import paths
@@ -57,7 +56,6 @@ planner.register_opts(CONF)
applier.register_opts(CONF)
decision_engine.register_opts(CONF)
maas_client.register_opts(CONF)
monasca_client.register_opts(CONF)
models.register_opts(CONF)
nova_client.register_opts(CONF)
gnocchi_client.register_opts(CONF)

6
watcher/conf/decision_engine.py
View File

@@ -71,10 +71,10 @@ WATCHER_DECISION_ENGINE_OPTS = [
'are the metric names as recognized by watcher and the '
'value is the real name of the metric in the datasource. '
'For example:: \n\n'
' monasca:\n'
' instance_cpu_usage: VM_CPU\n'
' gnocchi:\n'
' instance_cpu_usage: cpu_vm_util\n\n'
' instance_cpu_usage: cpu_vm_util\n'
' aetos:\n'
' instance_cpu_usage: ceilometer_cpu\n\n'
'This file is optional.'),
cfg.IntOpt('continuous_audit_interval',
default=10,

43
watcher/conf/monasca_client.py
View File

@@ -1,43 +0,0 @@
# Copyright (c) 2016 Intel Corp
#
# Authors: Prudhvi Rao Shedimbi <prudhvi.rao.shedimbi@intel.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
monasca_client = cfg.OptGroup(name='monasca_client',
title='Configuration Options for Monasca')
MONASCA_CLIENT_OPTS = [
cfg.StrOpt('api_version',
default='2_0',
help='Version of Monasca API to use in monascaclient.'),
cfg.StrOpt('interface',
default='internal',
choices=['public', 'internal', 'admin',
'publicURL', 'internalURL', 'adminURL'],
help='Type of interface used for monasca endpoint.'),
cfg.StrOpt('region_name',
help='Region in Identity service catalog to use for '
'communication with the OpenStack service.')]
def register_opts(conf):
conf.register_group(monasca_client)
conf.register_opts(MONASCA_CLIENT_OPTS, group=monasca_client)
def list_opts():
return [(monasca_client, MONASCA_CLIENT_OPTS)]

16
watcher/decision_engine/datasources/manager.py
View File

@@ -23,7 +23,6 @@ from watcher.common import exception
from watcher.decision_engine.datasources import aetos
from watcher.decision_engine.datasources import gnocchi as gnoc
from watcher.decision_engine.datasources import grafana as graf
from watcher.decision_engine.datasources import monasca as mon
from watcher.decision_engine.datasources import prometheus as prom
LOG = log.getLogger(__name__)
@@ -33,7 +32,6 @@ class DataSourceManager:
metric_map = OrderedDict([
(gnoc.GnocchiHelper.NAME, gnoc.GnocchiHelper.METRIC_MAP),
(mon.MonascaHelper.NAME, mon.MonascaHelper.METRIC_MAP),
(graf.GrafanaHelper.NAME, graf.GrafanaHelper.METRIC_MAP),
(prom.PrometheusHelper.NAME, prom.PrometheusHelper.METRIC_MAP),
(aetos.AetosHelper.NAME, aetos.AetosHelper.METRIC_MAP),
@@ -45,7 +43,6 @@ class DataSourceManager:
def __init__(self, config=None, osc=None):
self.osc = osc
self.config = config
self._monasca = None
self._gnocchi = None
self._grafana = None
self._prometheus = None
@@ -65,9 +62,6 @@ class DataSourceManager:
LOG.warning('Invalid Datasource: %s. Allowed: %s ', *msgargs)
self.datasources = self.config.datasources
if self.datasources and 'monasca' in self.datasources:
LOG.warning('The monasca datasource is deprecated and will be '
'removed in a future release.')
self._validate_datasource_config()
@@ -87,16 +81,6 @@ class DataSourceManager:
datasource_two=aetos.AetosHelper.NAME
)
@property
def monasca(self):
if self._monasca is None:
self._monasca = mon.MonascaHelper(osc=self.osc)
return self._monasca
@monasca.setter
def monasca(self, monasca):
self._monasca = monasca
@property
def gnocchi(self):
if self._gnocchi is None:

204
watcher/decision_engine/datasources/monasca.py
View File

@@ -1,204 +0,0 @@
# Copyright (c) 2016 b<>com
#
# Authors: Vincent FRANCOISE <vincent.francoise@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
from oslo_utils import timeutils
from watcher.common import clients
from watcher.decision_engine.datasources import base
class MonascaHelper(base.DataSourceBase):
NAME = 'monasca'
METRIC_MAP = dict(host_cpu_usage='cpu.percent',
host_ram_usage=None,
host_outlet_temp=None,
host_inlet_temp=None,
host_airflow=None,
host_power=None,
instance_cpu_usage='vm.cpu.utilization_perc',
instance_ram_usage=None,
instance_ram_allocated=None,
instance_l3_cache_usage=None,
instance_root_disk_size=None,
)
def __init__(self, osc=None):
":param osc: an OpenStackClients instance"
self.osc = osc if osc else clients.OpenStackClients()
self._monasca = None
@property
def monasca(self):
if self._monasca is None:
self._monasca = self.osc.monasca()
return self._monasca
def _format_time_params(self, start_time, end_time, period):
"""Format time-related params to the correct Monasca format
:param start_time: Start datetime from which metrics will be used
:param end_time: End datetime from which metrics will be used
:param period: interval in seconds (int)
:return: start ISO time, end ISO time, period
"""
if not period:
period = int(datetime.timedelta(hours=3).total_seconds())
if not start_time:
start_time = (
timeutils.utcnow() - datetime.timedelta(seconds=period))
start_timestamp = None if not start_time else start_time.isoformat()
end_timestamp = None if not end_time else end_time.isoformat()
return start_timestamp, end_timestamp, period
def query_retry_reset(self, exception_instance):
try:
from monascaclient import exc as mon_exc
except Exception:
mon_exc = None
if mon_exc and isinstance(exception_instance, mon_exc.Unauthorized):
self.osc.reset_clients()
self._monasca = None
def check_availability(self):
result = self.query_retry(self.monasca.metrics.list)
if result:
return 'available'
else:
return 'not available'
def list_metrics(self):
# TODO(alexchadin): this method should be implemented in accordance to
# monasca API.
pass
def statistic_aggregation(self, resource=None, resource_type=None,
meter_name=None, period=300, aggregate='mean',
granularity=300):
stop_time = timeutils.utcnow()
start_time = stop_time - datetime.timedelta(seconds=(int(period)))
meter = self._get_meter(meter_name)
if aggregate == 'mean':
aggregate = 'avg'
raw_kwargs = dict(
name=meter,
start_time=start_time.isoformat(),
end_time=stop_time.isoformat(),
dimensions={'hostname': resource.uuid},
period=period,
statistics=aggregate,
group_by='*',
)
kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
statistics = self.query_retry(
f=self.monasca.metrics.list_statistics, **kwargs)
cpu_usage = None
for stat in statistics:
avg_col_idx = stat['columns'].index(aggregate)
values = [r[avg_col_idx] for r in stat['statistics']]
value = float(sum(values)) / len(values)
cpu_usage = value
return cpu_usage
def statistic_series(self, resource=None, resource_type=None,
meter_name=None, start_time=None, end_time=None,
granularity=300):
meter = self._get_meter(meter_name)
raw_kwargs = dict(
name=meter,
start_time=start_time.isoformat(),
end_time=end_time.isoformat(),
dimensions={'hostname': resource.uuid},
statistics='avg',
group_by='*',
)
kwargs = {k: v for k, v in raw_kwargs.items() if k and v}
statistics = self.query_retry(
f=self.monasca.metrics.list_statistics, **kwargs)
result = {}
for stat in statistics:
v_index = stat['columns'].index('avg')
t_index = stat['columns'].index('timestamp')
result.update({r[t_index]: r[v_index] for r in stat['statistics']})
return result
def get_host_cpu_usage(self, resource, period,
aggregate, granularity=None):
return self.statistic_aggregation(
resource, 'compute_node', 'host_cpu_usage', period, aggregate,
granularity)
def get_host_ram_usage(self, resource, period,
aggregate, granularity=None):
raise NotImplementedError
def get_host_outlet_temp(self, resource, period,
aggregate, granularity=None):
raise NotImplementedError
def get_host_inlet_temp(self, resource, period,
aggregate, granularity=None):
raise NotImplementedError
def get_host_airflow(self, resource, period,
aggregate, granularity=None):
raise NotImplementedError
def get_host_power(self, resource, period,
aggregate, granularity=None):
raise NotImplementedError
def get_instance_cpu_usage(self, resource, period,
aggregate, granularity=None):
return self.statistic_aggregation(
resource, 'instance', 'instance_cpu_usage', period, aggregate,
granularity)
def get_instance_ram_usage(self, resource, period,
aggregate, granularity=None):
raise NotImplementedError
def get_instance_ram_allocated(self, resource, period,
aggregate, granularity=None):
raise NotImplementedError
def get_instance_l3_cache_usage(self, resource, period,
aggregate, granularity=None):
raise NotImplementedError
def get_instance_root_disk_size(self, resource, period,
aggregate, granularity=None):
raise NotImplementedError

57
watcher/tests/unit/common/test_clients.py
View File

@@ -10,7 +10,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from unittest import mock
@@ -22,11 +21,6 @@ from ironicclient import client as irclient
from ironicclient.v1 import client as irclient_v1
from keystoneauth1 import adapter as ka_adapter
from keystoneauth1 import loading as ka_loading
try:
from monascaclient.v2_0 import client as monclient_v2
MONASCA_INSTALLED = True
except Exception: # ImportError or others
MONASCA_INSTALLED = False
from novaclient import client as nvclient
from watcher.common import clients
@@ -229,57 +223,6 @@ class TestClients(base.TestCase):
cinder_cached = osc.cinder()
self.assertEqual(cinder, cinder_cached)
@unittest.skipUnless(MONASCA_INSTALLED, "requires python-monascaclient")
@mock.patch('monascaclient.client.Client')
@mock.patch.object(ka_loading, 'load_session_from_conf_options')
def test_clients_monasca(self, mock_session, mock_call):
mock_session.return_value = mock.Mock(
get_endpoint=mock.Mock(return_value='test_endpoint'),
get_token=mock.Mock(return_value='test_token'),)
self._register_watcher_clients_auth_opts()
osc = clients.OpenStackClients()
osc._monasca = None
osc.monasca()
mock_call.assert_called_once_with(
CONF.monasca_client.api_version,
'test_endpoint',
auth_url='http://server.ip:5000', cert_file=None, insecure=False,
key_file=None, keystone_timeout=None, os_cacert=None,
password='foopassword', service_type='monitoring',
token='test_token', username='foousername')
@unittest.skipUnless(MONASCA_INSTALLED, "requires python-monascaclient")
@mock.patch.object(ka_loading, 'load_session_from_conf_options')
def test_clients_monasca_diff_vers(self, mock_session):
mock_session.return_value = mock.Mock(
get_endpoint=mock.Mock(return_value='test_endpoint'),
get_token=mock.Mock(return_value='test_token'),)
self._register_watcher_clients_auth_opts()
CONF.set_override('api_version', '2_0', group='monasca_client')
osc = clients.OpenStackClients()
osc._monasca = None
osc.monasca()
self.assertEqual(monclient_v2.Client, type(osc.monasca()))
@unittest.skipUnless(MONASCA_INSTALLED, "requires python-monascaclient")
@mock.patch.object(ka_loading, 'load_session_from_conf_options')
def test_clients_monasca_cached(self, mock_session):
mock_session.return_value = mock.Mock(
get_endpoint=mock.Mock(return_value='test_endpoint'),
get_token=mock.Mock(return_value='test_token'),)
self._register_watcher_clients_auth_opts()
osc = clients.OpenStackClients()
osc._monasca = None
monasca = osc.monasca()
monasca_cached = osc.monasca()
self.assertEqual(monasca, monasca_cached)
@mock.patch.object(irclient, 'Client')
@mock.patch.object(clients.OpenStackClients, 'session')
def test_clients_ironic(self, mock_session, mock_call):

2
watcher/tests/unit/conf/test_list_opts.py
View File

@@ -38,7 +38,7 @@ class TestListOpts(base.TestCase):
'watcher_applier', 'watcher_datasources', 'watcher_planner',
'nova_client', 'gnocchi_client', 'grafana_client',
'grafana_translators', 'cinder_client',
'monasca_client', 'ironic_client', 'keystone_client',
'ironic_client', 'keystone_client',
'neutron_client', 'watcher_clients_auth', 'collector',
'placement_client']
self.opt_sections = list(dict(opts.list_opts()).keys())

45
watcher/tests/unit/decision_engine/datasources/test_manager.py
View File

@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from unittest import mock
from unittest.mock import MagicMock
@@ -24,13 +23,6 @@ from watcher.decision_engine.datasources import gnocchi
from watcher.decision_engine.datasources import grafana
from watcher.decision_engine.datasources import manager as ds_manager
from watcher.decision_engine.datasources import prometheus
try:
from monascaclient import client as monclient # noqa: F401
# Import monasca helper only when monasca client is installed
from watcher.decision_engine.datasources import monasca
MONASCA_INSTALLED = True
except Exception:
MONASCA_INSTALLED = False
from watcher.tests.unit import base
@@ -38,8 +30,6 @@ class TestDataSourceManager(base.BaseTestCase):
def _dsm_config(self, **kwargs):
dss = ['gnocchi']
if MONASCA_INSTALLED:
dss.append('monasca')
opts = dict(datasources=dss, metric_map_path=None)
opts.update(kwargs)
return MagicMock(**opts)
@@ -56,19 +46,6 @@ class TestDataSourceManager(base.BaseTestCase):
self.assertEqual(expected, actual)
self.assertEqual({}, manager.load_metric_map('/nope/nope/nope.yaml'))
@unittest.skipUnless(MONASCA_INSTALLED, "requires python-monascaclient")
def test_metric_file_metric_override(self):
path = 'watcher.decision_engine.datasources.manager.' \
'DataSourceManager.load_metric_map'
retval = {
monasca.MonascaHelper.NAME: {"host_airflow": "host_fnspid"}
}
with mock.patch(path, return_value=retval):
dsmcfg = self._dsm_config(datasources=['monasca'])
manager = self._dsm(config=dsmcfg)
backend = manager.get_backend(['host_airflow'])
self.assertEqual("host_fnspid", backend.METRIC_MAP['host_airflow'])
@mock.patch.object(grafana, 'CONF')
def test_metric_file_metric_override_grafana(self, m_config):
"""Grafana requires a different structure in the metric map"""
@@ -109,11 +86,11 @@ class TestDataSourceManager(base.BaseTestCase):
self.assertEqual(backend, manager.gnocchi)
def test_get_backend_order(self):
dss = ['monasca', 'gnocchi'] if MONASCA_INSTALLED else ['gnocchi']
dss = ['gnocchi']
dsmcfg = self._dsm_config(datasources=dss)
manager = self._dsm(config=dsmcfg)
backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
expected = manager.monasca if MONASCA_INSTALLED else manager.gnocchi
expected = manager.gnocchi
self.assertEqual(backend, expected)
def test_get_backend_wrong_metric(self):
@@ -128,17 +105,11 @@ class TestDataSourceManager(base.BaseTestCase):
def test_get_backend_error_datasource(self, m_gnocchi):
m_gnocchi.side_effect = exception.DataSourceNotAvailable
manager = self._dsm()
if MONASCA_INSTALLED:
backend = manager.get_backend(
['host_cpu_usage', 'instance_cpu_usage']
)
self.assertEqual(backend, manager.monasca)
else:
self.assertRaises(
exception.MetricNotAvailable,
manager.get_backend,
['host_cpu_usage', 'instance_cpu_usage']
)
self.assertRaises(
exception.MetricNotAvailable,
manager.get_backend,
['host_cpu_usage', 'instance_cpu_usage']
)
@mock.patch.object(grafana.GrafanaHelper, 'METRIC_MAP',
{'host_cpu_usage': 'test'})
@@ -230,8 +201,6 @@ class TestDataSourceManager(base.BaseTestCase):
aetos.AetosHelper.NAME,
gnocchi.GnocchiHelper.NAME,
]
if MONASCA_INSTALLED:
mixed_datasources.append(monasca.MonascaHelper.NAME)
dsmcfg = self._dsm_config(datasources=mixed_datasources)
# Should not raise any exception

162
watcher/tests/unit/decision_engine/datasources/test_monasca_helper.py
View File

@@ -1,162 +0,0 @@
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from datetime import datetime
from unittest import mock
from oslo_config import cfg
from watcher.common import clients
from watcher.common import exception
try:
from monascaclient import client as monclient # noqa: F401
from watcher.decision_engine.datasources import monasca as monasca_helper
MONASCA_INSTALLED = True
except Exception:
MONASCA_INSTALLED = False
from watcher.tests.unit import base
CONF = cfg.CONF
@mock.patch.object(clients.OpenStackClients, 'monasca')
@unittest.skipUnless(MONASCA_INSTALLED, "requires python-monascaclient")
class TestMonascaHelper(base.BaseTestCase):
def setUp(self):
super().setUp()
self.osc_mock = mock.Mock()
self.helper = monasca_helper.MonascaHelper(osc=self.osc_mock)
stat_agg_patcher = mock.patch.object(
self.helper, 'statistic_aggregation',
spec=monasca_helper.MonascaHelper.statistic_aggregation)
self.mock_aggregation = stat_agg_patcher.start()
self.addCleanup(stat_agg_patcher.stop)
def test_monasca_statistic_aggregation(self, mock_monasca):
monasca = mock.MagicMock()
expected_stat = [{
'columns': ['timestamp', 'avg'],
'dimensions': {
'hostname': 'rdev-indeedsrv001',
'service': 'monasca'},
'id': '0',
'name': 'cpu.percent',
'statistics': [
['2016-07-29T12:45:00Z', 0.0],
['2016-07-29T12:50:00Z', 0.9],
['2016-07-29T12:55:00Z', 0.9]]}]
monasca.metrics.list_statistics.return_value = expected_stat
mock_monasca.return_value = monasca
helper = monasca_helper.MonascaHelper()
result = helper.statistic_aggregation(
resource=mock.Mock(id='NODE_UUID'),
resource_type='compute_node',
meter_name='host_cpu_usage',
period=7200,
granularity=300,
aggregate='mean',
)
self.assertEqual(0.6, result)
def test_monasca_statistic_series(self, mock_monasca):
monasca = mock.MagicMock()
expected_stat = [{
'columns': ['timestamp', 'avg'],
'dimensions': {
'hostname': 'rdev-indeedsrv001',
'service': 'monasca'},
'id': '0',
'name': 'cpu.percent',
'statistics': [
['2016-07-29T12:45:00Z', 0.0],
['2016-07-29T12:50:00Z', 0.9],
['2016-07-29T12:55:00Z', 0.9]]}]
expected_result = {
'2016-07-29T12:45:00Z': 0.0,
'2016-07-29T12:50:00Z': 0.9,
'2016-07-29T12:55:00Z': 0.9,
}
monasca.metrics.list_statistics.return_value = expected_stat
mock_monasca.return_value = monasca
start = datetime(year=2016, month=7, day=29, hour=12, minute=45)
end = datetime(year=2016, month=7, day=29, hour=12, minute=55)
helper = monasca_helper.MonascaHelper()
result = helper.statistic_series(
resource=mock.Mock(id='NODE_UUID'),
resource_type='compute_node',
meter_name='host_cpu_usage',
start_time=start,
end_time=end,
granularity=300,
)
self.assertEqual(expected_result, result)
def test_statistic_aggregation_metric_unavailable(self, mock_monasca):
helper = monasca_helper.MonascaHelper()
# invalidate host_cpu_usage in metric map
original_metric_value = helper.METRIC_MAP.get('host_cpu_usage')
helper.METRIC_MAP.update(
host_cpu_usage=None
)
self.assertRaises(
exception.MetricNotAvailable, helper.statistic_aggregation,
resource=mock.Mock(id='NODE_UUID'), resource_type='compute_node',
meter_name='host_cpu_usage', period=7200, granularity=300,
aggregate='mean',
)
# restore the metric map as it is a static attribute that does not get
# restored between unit tests!
helper.METRIC_MAP.update(
instance_cpu_usage=original_metric_value
)
def test_check_availability(self, mock_monasca):
monasca = mock.MagicMock()
monasca.metrics.list.return_value = True
mock_monasca.return_value = monasca
helper = monasca_helper.MonascaHelper()
result = helper.check_availability()
self.assertEqual('available', result)
def test_check_availability_with_failure(self, mock_monasca):
monasca = mock.MagicMock()
monasca.metrics.list.side_effect = Exception()
mock_monasca.return_value = monasca
helper = monasca_helper.MonascaHelper()
self.assertEqual('not available', helper.check_availability())
def test_get_host_cpu_usage(self, mock_monasca):
self.mock_aggregation.return_value = 0.6
node = mock.Mock(id='compute1')
cpu_usage = self.helper.get_host_cpu_usage(node, 600, 'mean')
self.assertEqual(0.6, cpu_usage)
def test_get_instance_cpu_usage(self, mock_monasca):
self.mock_aggregation.return_value = 0.6
node = mock.Mock(id='vm1')
cpu_usage = self.helper.get_instance_cpu_usage(node, 600, 'mean')
self.assertEqual(0.6, cpu_usage)

132
watcher/tests/unit/decision_engine/model/monasca_metrics.py
View File

@@ -1,132 +0,0 @@
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class FakeMonascaMetrics:
def __init__(self):
self.emptytype = ""
def empty_one_metric(self, emptytype):
self.emptytype = emptytype
def mock_get_statistics(self, resource=None, resource_type=None,
meter_name=None, period=None, aggregate='mean',
granularity=None):
result = 0.0
if meter_name == 'host_cpu_usage':
result = self.get_usage_compute_node_cpu(resource)
elif meter_name == 'instance_cpu_usage':
result = self.get_average_usage_instance_cpu(resource)
return result
def mock_get_statistics_wb(self, resource=None, resource_type=None,
meter_name=None, period=None, aggregate='mean',
granularity=None):
"""Statistics for workload balance strategy"""
result = 0.0
if meter_name == 'instance_cpu_usage':
result = self.get_average_usage_instance_cpu_wb(resource)
return result
@staticmethod
def get_usage_compute_node_cpu(*args, **kwargs):
"""The last VM CPU usage values to average
:param uuid:00
:return:
"""
resource = args[0]
uuid = resource.uuid
measurements = {}
# node 0
measurements['Node_0'] = 7
measurements['Node_1'] = 7
# node 1
measurements['Node_2'] = 80
# node 2
measurements['Node_3'] = 5
measurements['Node_4'] = 5
measurements['Node_5'] = 10
# node 3
measurements['Node_6'] = 8
measurements['Node_19'] = 10
# node 4
measurements['INSTANCE_7'] = 4
if uuid not in measurements.keys():
# measurements[uuid] = random.randint(1, 4)
measurements[uuid] = 8
statistics = [
{'columns': ['avg'],
'statistics': [[float(measurements[str(uuid)])]]}]
cpu_usage = None
for stat in statistics:
avg_col_idx = stat['columns'].index('avg')
values = [r[avg_col_idx] for r in stat['statistics']]
value = float(sum(values)) / len(values)
cpu_usage = value
return cpu_usage
@staticmethod
def get_average_usage_instance_cpu(*args, **kwargs):
"""The last VM CPU usage values to average
:param uuid:00
:return:
"""
resource = args[0]
uuid = resource.uuid
measurements = {}
# node 0
measurements['INSTANCE_0'] = 7
measurements['d000ef1f-dc19-4982-9383-087498bfde03'] = 7
measurements['d010ef1f-dc19-4982-9383-087498bfde03'] = 7
measurements['INSTANCE_1'] = 7
# node 1
measurements['d020ef1f-dc19-4982-9383-087498bfde03'] = 10
measurements['INSTANCE_2'] = 10
# node 2
measurements['INSTANCE_3'] = 5
measurements['INSTANCE_4'] = 5
measurements['INSTANCE_5'] = 10
# node 3
measurements['d060ef1f-dc19-4982-9383-087498bfde03'] = 8
measurements['INSTANCE_6'] = 8
# node 4
measurements['d070ef1f-dc19-4982-9383-087498bfde03'] = 4
measurements['INSTANCE_7'] = 4
if uuid not in measurements.keys():
# measurements[uuid] = random.randint(1, 4)
measurements[uuid] = 8
statistics = [
{'columns': ['avg'],
'statistics': [[float(measurements[str(uuid)])]]}]
cpu_usage = None
for stat in statistics:
avg_col_idx = stat['columns'].index('avg')
values = [r[avg_col_idx] for r in stat['statistics']]
value = float(sum(values)) / len(values)
cpu_usage = value
return cpu_usage

20
watcher/tests/unit/decision_engine/strategy/strategies/test_base.py
View File

@@ -22,14 +22,6 @@ from watcher.decision_engine.strategy import strategies
from watcher.tests.unit import base
from watcher.tests.unit.decision_engine.model import faker_cluster_state
try:
# Only check availability; tests mock DataSourceManager so this is just
# to build conditional example lists below.
from monascaclient import client as monclient # noqa: F401
MONASCA_INSTALLED = True
except Exception:
MONASCA_INSTALLED = False
class TestBaseStrategy(base.TestCase):
@@ -70,8 +62,6 @@ class TestBaseStrategyDatasource(TestBaseStrategy):
def test_global_preference(self, m_conf, m_manager):
"""Test if the global preference is used"""
dss = ['gnocchi']
if MONASCA_INSTALLED:
dss.append('monasca')
m_conf.watcher_datasources.datasources = dss
# Make sure we access the property and not the underlying function.
@@ -90,10 +80,7 @@ class TestBaseStrategyDatasource(TestBaseStrategy):
@mock.patch.object(strategies.base, 'CONF')
def test_global_preference_reverse(self, m_conf, m_manager):
"""Test if the global preference is used with another order"""
if MONASCA_INSTALLED:
dss = ['monasca', 'gnocchi']
else:
dss = ['gnocchi']
dss = ['gnocchi']
m_conf.watcher_datasources.datasources = dss
# Make sure we access the property and not the underlying function.
@@ -118,10 +105,7 @@ class TestBaseStrategyDatasource(TestBaseStrategy):
self.strategy = strategies.DummyStrategy(
config=datasources)
if MONASCA_INSTALLED:
m_conf.watcher_datasources.datasources = ['monasca', 'gnocchi']
else:
m_conf.watcher_datasources.datasources = ['gnocchi']
m_conf.watcher_datasources.datasources = ['gnocchi']
# Access the property so that the configuration is read in order to
# get the correct datasource

4
watcher/tests/unit/decision_engine/strategy/strategies/test_basic_consolidation.py
View File

@@ -24,7 +24,6 @@ from watcher.common import clients
from watcher.decision_engine.model import model_root
from watcher.decision_engine.strategy import strategies
from watcher.tests.unit.decision_engine.model import gnocchi_metrics
from watcher.tests.unit.decision_engine.model import monasca_metrics
from watcher.tests.unit.decision_engine.strategy.strategies.test_base \
import TestBaseStrategy
@@ -32,9 +31,6 @@ from watcher.tests.unit.decision_engine.strategy.strategies.test_base \
class TestBasicConsolidation(TestBaseStrategy):
scenarios = [
("Monasca",
{"datasource": "monasca",
"fake_datasource_cls": monasca_metrics.FakeMonascaMetrics}),
("Gnocchi",
{"datasource": "gnocchi",
"fake_datasource_cls": gnocchi_metrics.FakeGnocchiMetrics}),