From e8faf3a2a6447e8ccf37fd570899dcfbeccdeb5c Mon Sep 17 00:00:00 2001 From: Lianhao Lu Date: Fri, 13 Dec 2013 14:24:35 +0800 Subject: [PATCH] Added a new scheduler filter for metrics The scheduler filter MetricsFilter filters out those hosts which don't have the metrics data available as required by metric settings. This is part of the blueprint utilization-aware-scheduling. DocImpact: Added a new metrics filter. Change-Id: Ib4a898774daf683c4496ef3e9953d23027f11ac0 --- doc/source/devref/filter_scheduler.rst | 3 ++ nova/scheduler/filters/metrics_filter.py | 54 ++++++++++++++++++++ nova/scheduler/utils.py | 33 ++++++++++++ nova/scheduler/weights/metrics.py | 26 ++-------- nova/tests/scheduler/test_host_filters.py | 16 ++++++ nova/tests/scheduler/test_scheduler_utils.py | 29 +++++++++++ 6 files changed, 140 insertions(+), 21 deletions(-) create mode 100644 nova/scheduler/filters/metrics_filter.py diff --git a/doc/source/devref/filter_scheduler.rst b/doc/source/devref/filter_scheduler.rst index 693d292f16f3..63f2b37d393b 100644 --- a/doc/source/devref/filter_scheduler.rst +++ b/doc/source/devref/filter_scheduler.rst @@ -109,6 +109,8 @@ There are some standard filter classes to use (:mod:`nova.scheduler.filters`): * |AggregateMultiTenancyIsolation| - isolate tenants in specific aggregates. * |AggregateImagePropertiesIsolation| - isolates hosts based on image properties and aggregate metadata. +* |MetricsFilter| - filters hosts based on metrics weight_setting. Only hosts with + the available metrics are passed. Now we can focus on these standard filter classes in details. I will pass the simplest ones, such as |AllHostsFilter|, |CoreFilter| and |RamFilter| are, @@ -334,3 +336,4 @@ in :mod:``nova.tests.scheduler``. .. |AggregateMultiTenancyIsolation| replace:: :class:`AggregateMultiTenancyIsolation ` .. |RamWeigher| replace:: :class:`RamWeigher ` .. |AggregateImagePropertiesIsolation| replace:: :class:`AggregateImagePropertiesIsolation ` +.. |MetricsFilter| replace:: :class:`MetricsFilter ` diff --git a/nova/scheduler/filters/metrics_filter.py b/nova/scheduler/filters/metrics_filter.py new file mode 100644 index 000000000000..60f6202f8e7d --- /dev/null +++ b/nova/scheduler/filters/metrics_filter.py @@ -0,0 +1,54 @@ +# Copyright (c) 2014 Intel, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg + +from nova.openstack.common.gettextutils import _ +from nova.openstack.common import log as logging +from nova.scheduler import filters +from nova.scheduler import utils + +LOG = logging.getLogger(__name__) + +CONF = cfg.CONF +CONF.import_opt('weight_setting', + 'nova.scheduler.weights.metrics', + group='metrics') + + +class MetricsFilter(filters.BaseHostFilter): + """Metrics Filter + + This filter is used to filter out those hosts which don't have the + corresponding metrics so these the metrics weigher won't fail due to + these hosts. + """ + + def __init__(self): + super(MetricsFilter, self).__init__() + opts = utils.parse_options(CONF.metrics.weight_setting, + sep='=', + converter=float, + name="metrics.weight_setting") + self.keys = [x[0] for x in opts] + + def host_passes(self, host_state, filter_properties): + unavail = [i for i in self.keys if i not in host_state.metrics] + if unavail: + LOG.debug(_("%(host_state)s does not have the following " + "metrics: %(metrics)s"), + {'host_state': host_state, + 'metrics': ', '.join(unavail)}) + return len(unavail) == 0 diff --git a/nova/scheduler/utils.py b/nova/scheduler/utils.py index 011e5b89585b..3ffef1054416 100644 --- a/nova/scheduler/utils.py +++ b/nova/scheduler/utils.py @@ -134,3 +134,36 @@ def _add_retry_host(filter_properties, host, node): return hosts = retry['hosts'] hosts.append([host, node]) + + +def parse_options(opts, sep='=', converter=str, name=""): + """Parse a list of options, each in the format of . Also + use the converter to convert the value into desired type. + + :params opts: list of options, e.g. from oslo.config.cfg.ListOpt + :params sep: the separator + :params converter: callable object to convert the value, should raise + ValueError for conversion failure + :params name: name of the option + + :returns: a lists of tuple of values (key, converted_value) + """ + good = [] + bad = [] + for opt in opts: + try: + key, seen_sep, value = opt.partition(sep) + value = converter(value) + except ValueError: + key = None + value = None + if key and seen_sep and value is not None: + good.append((key, value)) + else: + bad.append(opt) + if bad: + LOG.warn(_("Ignoring the invalid elements of the option " + "%(name)s: %(options)s"), + {'name': name, + 'options': ", ".join(bad)}) + return good diff --git a/nova/scheduler/weights/metrics.py b/nova/scheduler/weights/metrics.py index 7dd2b78f32ea..27e9fb9d5aab 100644 --- a/nova/scheduler/weights/metrics.py +++ b/nova/scheduler/weights/metrics.py @@ -28,8 +28,7 @@ in the configuration file as the followings: from oslo.config import cfg from nova import exception -from nova.openstack.common.gettextutils import _ -from nova.openstack.common import log as logging +from nova.scheduler import utils from nova.scheduler import weights metrics_weight_opts = [ @@ -50,31 +49,16 @@ metrics_weight_opts = [ CONF = cfg.CONF CONF.register_opts(metrics_weight_opts, group='metrics') -LOG = logging.getLogger(__name__) - class MetricsWeigher(weights.BaseHostWeigher): def __init__(self): self._parse_setting() def _parse_setting(self): - self.setting = [] - bad = [] - for item in CONF.metrics.weight_setting: - try: - (name, ratio) = item.split('=') - ratio = float(ratio) - except ValueError: - name = None - ratio = None - if name and ratio is not None: - self.setting.append((name, ratio)) - else: - bad.append(item) - if bad: - LOG.error(_("Ignoring the invalid elements of" - " metrics_weight_setting: %s"), - ",".join(bad)) + self.setting = utils.parse_options(CONF.metrics.weight_setting, + sep='=', + converter=float, + name="metrics.weight_setting") def weight_multiplier(self): """Override the weight multiplier.""" diff --git a/nova/tests/scheduler/test_host_filters.py b/nova/tests/scheduler/test_host_filters.py index c5e11ccbb42c..9e706c32f80a 100644 --- a/nova/tests/scheduler/test_host_filters.py +++ b/nova/tests/scheduler/test_host_filters.py @@ -1745,3 +1745,19 @@ class HostFiltersTestCase(test.NoDBTestCase): 'foo2': 'bar3'}}}} host = fakes.FakeHostState('host1', 'compute', {}) self.assertTrue(filt_cls.host_passes(host, filter_properties)) + + def test_metrics_filter_pass(self): + self.flags(weight_setting=['foo=1', 'bar=2'], group='metrics') + metrics = dict(foo=1, bar=2) + host = fakes.FakeHostState('host1', 'node1', + attribute_dict={'metrics': metrics}) + filt_cls = self.class_map['MetricsFilter']() + self.assertTrue(filt_cls.host_passes(host, None)) + + def test_metrics_filter_missing_metrics(self): + self.flags(weight_setting=['foo=1', 'bar=2'], group='metrics') + metrics = dict(foo=1) + host = fakes.FakeHostState('host1', 'node1', + attribute_dict={'metrics': metrics}) + filt_cls = self.class_map['MetricsFilter']() + self.assertFalse(filt_cls.host_passes(host, None)) diff --git a/nova/tests/scheduler/test_scheduler_utils.py b/nova/tests/scheduler/test_scheduler_utils.py index 44dd06b4877b..da42cc074ba0 100644 --- a/nova/tests/scheduler/test_scheduler_utils.py +++ b/nova/tests/scheduler/test_scheduler_utils.py @@ -189,3 +189,32 @@ class SchedulerUtilsTestCase(test.NoDBTestCase): def test_populate_filter_props_force_nodes_no_retry(self): self._test_populate_filter_props(force_nodes=['force-node']) + + def _check_parse_options(self, opts, sep, converter, expected): + good = scheduler_utils.parse_options(opts, + sep=sep, + converter=converter) + for item in expected: + self.assertIn(item, good) + + def test_parse_options(self): + # check normal + self._check_parse_options(['foo=1', 'bar=-2.1'], + '=', + float, + [('foo', 1.0), ('bar', -2.1)]) + # check convert error + self._check_parse_options(['foo=a1', 'bar=-2.1'], + '=', + float, + [('bar', -2.1)]) + # check seperator missing + self._check_parse_options(['foo', 'bar=-2.1'], + '=', + float, + [('bar', -2.1)]) + # check key missing + self._check_parse_options(['=5', 'bar=-2.1'], + '=', + float, + [('bar', -2.1)])