MetricsWeigher: Added support of unavailable metrics

This patch added the support of unavailable metrics in the metric
weigher. Based on the configuration, if an unavailable metric is met,
the metric weigher might either throw an exception, or return an
configurable weight value.

This is part of the blueprint utilization-aware-scheduling.

DocImpact: Added support of unavailable metrics. 2 new config options
are added, 1) 'required' is a boolean option indicating whether all
the metrics set by 'weight_setting' are needed to be available when
calculating the weight. 2) 'weight_of_unavailable' is a float option
specifying the value to be returned if the metrics is unavailable. See
etc/nova/nova.conf.sample for details.

Change-Id: Icb951605aae28487b00a12d172a3ac4dafda2cdb

etc/nova/nova.conf.sample

@@ -3030,6 +3030,21 @@
 # -1.0. (list value)
 #weight_setting=
 
+# How to treat the unavailable metrics. When a metric is NOT
+# available for a host, if it is set to be True, it would
+# raise an exception, so it is recommended to use the
+# scheduler filter MetricFilter to filter out those hosts. If
+# it is set to be False, the unavailable metric would be
+# treated as a negative factor in weighing process, the
+# returned value would be set by the option
+# weight_of_unavailable. (boolean value)
+#required=true
+
+# The final weight value to be returned if required is set to
+# False and any one of the metrics set by weight_setting is
+# unavailable. (floating point value)
+#weight_of_unavailable=-10000.0
+
 
 [osapi_v3]
 

nova/scheduler/weights/metrics.py

@@ -20,7 +20,8 @@ This weigher can compute the weight based on the compute node host's various
 metrics. The to-be weighed metrics and their weighing ratio are specified
 in the configuration file as the followings:
 
-    metrics_weight_setting = name1=1.0, name2=-1.0
+    [metrics]
+    weight_setting = name1=1.0, name2=-1.0
 
     The final weight would be name1.value * 1.0 + name2.value * -1.0.
 """
@@ -44,6 +45,22 @@ metrics_weight_opts = [
                          'the corresponding ratio. So for "name1=1.0, '
                          'name2=-1.0" The final weight would be '
                          'name1.value * 1.0 + name2.value * -1.0.'),
+       cfg.BoolOpt('required',
+                    default=True,
+                    help='How to treat the unavailable metrics. When a '
+                         'metric is NOT available for a host, if it is set '
+                         'to be True, it would raise an exception, so it '
+                         'is recommended to use the scheduler filter '
+                         'MetricFilter to filter out those hosts. If it is '
+                         'set to be False, the unavailable metric would be '
+                         'treated as a negative factor in weighing '
+                         'process, the returned value would be set by '
+                         'the option weight_of_unavailable.'),
+       cfg.FloatOpt('weight_of_unavailable',
+                     default=float(-10000.0),
+                     help='The final weight value to be returned if '
+                          'required is set to False and any one of the '
+                          'metrics set by weight_setting is unavailable.'),
 ]
 
 CONF = cfg.CONF
@@ -71,8 +88,17 @@ class MetricsWeigher(weights.BaseHostWeigher):
             try:
                 value += host_state.metrics[name].value * ratio
             except KeyError:
-                raise exception.ComputeHostMetricNotFound(
-                        host=host_state.host,
-                        node=host_state.nodename,
-                        name=name)
+                if CONF.metrics.required:
+                    raise exception.ComputeHostMetricNotFound(
+                            host=host_state.host,
+                            node=host_state.nodename,
+                            name=name)
+                else:
+                    # We treat the unavailable metric as the most negative
+                    # factor, i.e. set the value to make this obj would be
+                    # at the end of the ordered weighed obj list
+                    # Do nothing if ratio or weight_multiplier is 0.
+                    if ratio * self.weight_multiplier() != 0:
+                        return CONF.metrics.weight_of_unavailable
+
         return value

nova/tests/scheduler/fakes.py

@@ -123,6 +123,50 @@ COMPUTE_NODES_METRICS = [
                                        'source': 'host4'
                                       },
                                      ])),
+        dict(id=5, local_gb=768, memory_mb=768, vcpus=8,
+             disk_available_least=768, free_ram_mb=768, vcpus_used=0,
+             free_disk_gb=768, local_gb_used=0, updated_at=None,
+             service=dict(host='host5', disabled=False),
+             hypervisor_hostname='node5', host_ip='127.0.0.1',
+             hypervisor_version=0,
+             metrics=jsonutils.dumps([{'name': 'foo',
+                                       'value': 768,
+                                       'timestamp': None,
+                                       'source': 'host5'
+                                      },
+                                      {'name': 'bar',
+                                       'value': 0,
+                                       'timestamp': None,
+                                       'source': 'host5'
+                                      },
+                                      {'name': 'zot',
+                                       'value': 1,
+                                       'timestamp': None,
+                                       'source': 'host5'
+                                      },
+                                     ])),
+        dict(id=6, local_gb=2048, memory_mb=2048, vcpus=8,
+             disk_available_least=2048, free_ram_mb=2048, vcpus_used=0,
+             free_disk_gb=2048, local_gb_used=0, updated_at=None,
+             service=dict(host='host6', disabled=False),
+             hypervisor_hostname='node6', host_ip='127.0.0.1',
+             hypervisor_version=0,
+             metrics=jsonutils.dumps([{'name': 'foo',
+                                       'value': 2048,
+                                       'timestamp': None,
+                                       'source': 'host6'
+                                      },
+                                      {'name': 'bar',
+                                       'value': 0,
+                                       'timestamp': None,
+                                       'source': 'host6'
+                                      },
+                                      {'name': 'zot',
+                                       'value': 2,
+                                       'timestamp': None,
+                                       'source': 'host6'
+                                      },
+                                     ])),
 ]
 
 INSTANCES = [

nova/tests/scheduler/test_weights.py

@@ -227,10 +227,22 @@ class MetricsWeigherTestCase(test.NoDBTestCase):
                                    ['=5', 'bar=-2.1'],
                                    [('bar', -2.1)])
 
-    def test_metric_not_found(self):
+    def test_metric_not_found_required(self):
         setting = ['foo=1', 'zot=2']
         self.assertRaises(exception.ComputeHostMetricNotFound,
                           self._do_test,
                           setting,
                           8192,
                           'host4')
+
+    def test_metric_not_found_non_required(self):
+        # host1: foo=512,  bar=1
+        # host2: foo=1024, bar=2
+        # host3: foo=3072, bar=1
+        # host4: foo=8192, bar=0
+        # host5: foo=768, bar=0, zot=1
+        # host6: foo=2048, bar=0, zot=2
+        # so, host5 should win:
+        self.flags(required=False, group='metrics')
+        setting = ['foo=0.0001', 'zot=-1']
+        self._do_test(setting, 1.0, 'host5')