Added a new scheduler filter for metrics

The scheduler filter MetricsFilter filters out those hosts which don't
have the metrics data available as required by metric settings.

This is part of the blueprint utilization-aware-scheduling.

DocImpact: Added a new metrics filter.

Change-Id: Ib4a898774daf683c4496ef3e9953d23027f11ac0

doc/source/devref/filter_scheduler.rst

@@ -109,6 +109,8 @@ There are some standard filter classes to use (:mod:`nova.scheduler.filters`):
 * |AggregateMultiTenancyIsolation| - isolate tenants in specific aggregates.
 * |AggregateImagePropertiesIsolation| - isolates hosts based on image
   properties and aggregate metadata.
+* |MetricsFilter| - filters hosts based on metrics weight_setting. Only hosts with
+  the available metrics are passed.
 
 Now we can focus on these standard filter classes in details. I will pass the
 simplest ones, such as |AllHostsFilter|, |CoreFilter| and |RamFilter| are,
@@ -334,3 +336,4 @@ in :mod:``nova.tests.scheduler``.
 .. |AggregateMultiTenancyIsolation| replace:: :class:`AggregateMultiTenancyIsolation <nova.scheduler.filters.aggregate_multitenancy_isolation.AggregateMultiTenancyIsolation>`
 .. |RamWeigher| replace:: :class:`RamWeigher <nova.scheduler.weights.all_weighers.RamWeigher>`
 .. |AggregateImagePropertiesIsolation| replace:: :class:`AggregateImagePropertiesIsolation <nova.scheduler.filters.aggregate_image_properties_isolation.AggregateImagePropertiesIsolation>`
+.. |MetricsFilter| replace:: :class:`MetricsFilter <nova.scheduler.filters.metrics_filter.MetricsFilter>`

nova/scheduler/filters/metrics_filter.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2014 Intel, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo.config import cfg
+
+from nova.openstack.common.gettextutils import _
+from nova.openstack.common import log as logging
+from nova.scheduler import filters
+from nova.scheduler import utils
+
+LOG = logging.getLogger(__name__)
+
+CONF = cfg.CONF
+CONF.import_opt('weight_setting',
+                'nova.scheduler.weights.metrics',
+                group='metrics')
+
+
+class MetricsFilter(filters.BaseHostFilter):
+    """Metrics Filter
+
+    This filter is used to filter out those hosts which don't have the
+    corresponding metrics so these the metrics weigher won't fail due to
+    these hosts.
+    """
+
+    def __init__(self):
+        super(MetricsFilter, self).__init__()
+        opts = utils.parse_options(CONF.metrics.weight_setting,
+                                   sep='=',
+                                   converter=float,
+                                   name="metrics.weight_setting")
+        self.keys = [x[0] for x in opts]
+
+    def host_passes(self, host_state, filter_properties):
+        unavail = [i for i in self.keys if i not in host_state.metrics]
+        if unavail:
+            LOG.debug(_("%(host_state)s does not have the following "
+                        "metrics: %(metrics)s"),
+                      {'host_state': host_state,
+                       'metrics': ', '.join(unavail)})
+        return len(unavail) == 0

nova/scheduler/utils.py

@@ -134,3 +134,36 @@ def _add_retry_host(filter_properties, host, node):
         return
     hosts = retry['hosts']
     hosts.append([host, node])
+
+
+def parse_options(opts, sep='=', converter=str, name=""):
+    """Parse a list of options, each in the format of <key><sep><value>. Also
+    use the converter to convert the value into desired type.
+
+    :params opts: list of options, e.g. from oslo.config.cfg.ListOpt
+    :params sep: the separator
+    :params converter: callable object to convert the value, should raise
+                       ValueError for conversion failure
+    :params name: name of the option
+
+    :returns: a lists of tuple of values (key, converted_value)
+    """
+    good = []
+    bad = []
+    for opt in opts:
+        try:
+            key, seen_sep, value = opt.partition(sep)
+            value = converter(value)
+        except ValueError:
+            key = None
+            value = None
+        if key and seen_sep and value is not None:
+            good.append((key, value))
+        else:
+            bad.append(opt)
+    if bad:
+        LOG.warn(_("Ignoring the invalid elements of the option "
+                   "%(name)s: %(options)s"),
+                {'name': name,
+                 'options': ", ".join(bad)})
+    return good

nova/scheduler/weights/metrics.py

@@ -28,8 +28,7 @@ in the configuration file as the followings:
 from oslo.config import cfg
 
 from nova import exception
-from nova.openstack.common.gettextutils import _
-from nova.openstack.common import log as logging
+from nova.scheduler import utils
 from nova.scheduler import weights
 
 metrics_weight_opts = [
@@ -50,31 +49,16 @@ metrics_weight_opts = [
 CONF = cfg.CONF
 CONF.register_opts(metrics_weight_opts, group='metrics')
 
-LOG = logging.getLogger(__name__)
-
 
 class MetricsWeigher(weights.BaseHostWeigher):
     def __init__(self):
         self._parse_setting()
 
     def _parse_setting(self):
-        self.setting = []
-        bad = []
-        for item in CONF.metrics.weight_setting:
-            try:
-                (name, ratio) = item.split('=')
-                ratio = float(ratio)
-            except ValueError:
-                name = None
-                ratio = None
-            if name and ratio is not None:
-                self.setting.append((name, ratio))
-            else:
-                bad.append(item)
-        if bad:
-            LOG.error(_("Ignoring the invalid elements of"
-                        " metrics_weight_setting: %s"),
-                      ",".join(bad))
+        self.setting = utils.parse_options(CONF.metrics.weight_setting,
+                                           sep='=',
+                                           converter=float,
+                                           name="metrics.weight_setting")
 
     def weight_multiplier(self):
         """Override the weight multiplier."""

nova/tests/scheduler/test_host_filters.py

@@ -1745,3 +1745,19 @@ class HostFiltersTestCase(test.NoDBTestCase):
                                                     'foo2': 'bar3'}}}}
         host = fakes.FakeHostState('host1', 'compute', {})
         self.assertTrue(filt_cls.host_passes(host, filter_properties))
+
+    def test_metrics_filter_pass(self):
+        self.flags(weight_setting=['foo=1', 'bar=2'], group='metrics')
+        metrics = dict(foo=1, bar=2)
+        host = fakes.FakeHostState('host1', 'node1',
+                                   attribute_dict={'metrics': metrics})
+        filt_cls = self.class_map['MetricsFilter']()
+        self.assertTrue(filt_cls.host_passes(host, None))
+
+    def test_metrics_filter_missing_metrics(self):
+        self.flags(weight_setting=['foo=1', 'bar=2'], group='metrics')
+        metrics = dict(foo=1)
+        host = fakes.FakeHostState('host1', 'node1',
+                                   attribute_dict={'metrics': metrics})
+        filt_cls = self.class_map['MetricsFilter']()
+        self.assertFalse(filt_cls.host_passes(host, None))

nova/tests/scheduler/test_scheduler_utils.py

@@ -189,3 +189,32 @@ class SchedulerUtilsTestCase(test.NoDBTestCase):
 
     def test_populate_filter_props_force_nodes_no_retry(self):
         self._test_populate_filter_props(force_nodes=['force-node'])
+
+    def _check_parse_options(self, opts, sep, converter, expected):
+        good = scheduler_utils.parse_options(opts,
+                                             sep=sep,
+                                             converter=converter)
+        for item in expected:
+            self.assertIn(item, good)
+
+    def test_parse_options(self):
+        # check normal
+        self._check_parse_options(['foo=1', 'bar=-2.1'],
+                                  '=',
+                                  float,
+                                  [('foo', 1.0), ('bar', -2.1)])
+        # check convert error
+        self._check_parse_options(['foo=a1', 'bar=-2.1'],
+                                  '=',
+                                  float,
+                                  [('bar', -2.1)])
+        # check seperator missing
+        self._check_parse_options(['foo', 'bar=-2.1'],
+                                  '=',
+                                  float,
+                                  [('bar', -2.1)])
+        # check key missing
+        self._check_parse_options(['=5', 'bar=-2.1'],
+                                  '=',
+                                  float,
+                                  [('bar', -2.1)])