Add DriverFilter and GoodnessWeigher to manila

This patch ports cinder's DriverFilter and GoodnessWeigher to manila.
These can use two new properties provided by backends,
'filter_function' and 'goodness_function', which can be used to filter
and weigh qualified backends, respectively.

Reference for cinder spec: I59b607a88953a346aa35e67e785a0417a7ce8cc9
Reference for cinder commit: I38408ab49b6ed869c1faae746ee64a3bae86be58

DocImpact
Change-Id: I873f4152e16efdeb30ceae26335a7974dc9b4b69
Implements: blueprint driver-filter-goodness-weigher

manila/exception.py

@@ -782,3 +782,7 @@ class TegileAPIException(ShareBackendException):
 
 class StorageCommunicationException(ShareBackendException):
     message = _("Could not communicate with storage array.")
+
+
+class EvaluatorParseException(ManilaException):
+    message = _("Error during evaluator parsing: %(reason)s")

manila/scheduler/evaluator/evaluator.py

@@ -0,0 +1,297 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import operator
+import re
+
+import pyparsing
+import six
+
+from manila import exception
+from manila.i18n import _
+
+
+def _operatorOperands(tokenList):
+    it = iter(tokenList)
+    while 1:
+        try:
+            op1 = next(it)
+            op2 = next(it)
+            yield(op1, op2)
+        except StopIteration:
+            break
+
+
+class EvalConstant(object):
+    def __init__(self, toks):
+        self.value = toks[0]
+
+    def eval(self):
+        result = self.value
+        if (isinstance(result, six.string_types) and
+                re.match("^[a-zA-Z_]+\.[a-zA-Z_]+$", result)):
+            (which_dict, entry) = result.split('.')
+            try:
+                result = _vars[which_dict][entry]
+            except KeyError as e:
+                msg = _("KeyError: %s") % six.text_type(e)
+                raise exception.EvaluatorParseException(reason=msg)
+            except TypeError as e:
+                msg = _("TypeError: %s") % six.text_type(e)
+                raise exception.EvaluatorParseException(reason=msg)
+
+        try:
+            result = int(result)
+        except ValueError:
+            try:
+                result = float(result)
+            except ValueError as e:
+                msg = _("ValueError: %s") % six.text_type(e)
+                raise exception.EvaluatorParseException(reason=msg)
+
+        return result
+
+
+class EvalSignOp(object):
+    operations = {
+        '+': 1,
+        '-': -1,
+    }
+
+    def __init__(self, toks):
+        self.sign, self.value = toks[0]
+
+    def eval(self):
+        return self.operations[self.sign] * self.value.eval()
+
+
+class EvalAddOp(object):
+    def __init__(self, toks):
+        self.value = toks[0]
+
+    def eval(self):
+        sum = self.value[0].eval()
+        for op, val in _operatorOperands(self.value[1:]):
+            if op == '+':
+                sum += val.eval()
+            elif op == '-':
+                sum -= val.eval()
+        return sum
+
+
+class EvalMultOp(object):
+    def __init__(self, toks):
+        self.value = toks[0]
+
+    def eval(self):
+        prod = self.value[0].eval()
+        for op, val in _operatorOperands(self.value[1:]):
+            try:
+                if op == '*':
+                    prod *= val.eval()
+                elif op == '/':
+                    prod /= float(val.eval())
+            except ZeroDivisionError as e:
+                msg = _("ZeroDivisionError: %s") % six.text_type(e)
+                raise exception.EvaluatorParseException(reason=msg)
+        return prod
+
+
+class EvalPowerOp(object):
+    def __init__(self, toks):
+        self.value = toks[0]
+
+    def eval(self):
+        prod = self.value[0].eval()
+        for op, val in _operatorOperands(self.value[1:]):
+            prod = pow(prod, val.eval())
+        return prod
+
+
+class EvalNegateOp(object):
+    def __init__(self, toks):
+        self.negation, self.value = toks[0]
+
+    def eval(self):
+        return not self.value.eval()
+
+
+class EvalComparisonOp(object):
+    operations = {
+        "<": operator.lt,
+        "<=": operator.le,
+        ">": operator.gt,
+        ">=": operator.ge,
+        "!=": operator.ne,
+        "==": operator.eq,
+        "<>": operator.ne,
+    }
+
+    def __init__(self, toks):
+        self.value = toks[0]
+
+    def eval(self):
+        val1 = self.value[0].eval()
+        for op, val in _operatorOperands(self.value[1:]):
+            fn = self.operations[op]
+            val2 = val.eval()
+            if not fn(val1, val2):
+                break
+            val1 = val2
+        else:
+            return True
+        return False
+
+
+class EvalTernaryOp(object):
+    def __init__(self, toks):
+        self.value = toks[0]
+
+    def eval(self):
+        condition = self.value[0].eval()
+        if condition:
+            return self.value[2].eval()
+        else:
+            return self.value[4].eval()
+
+
+class EvalFunction(object):
+    functions = {
+        "abs": abs,
+        "max": max,
+        "min": min,
+    }
+
+    def __init__(self, toks):
+        self.func, self.value = toks[0]
+
+    def eval(self):
+        args = self.value.eval()
+        if type(args) is list:
+            return self.functions[self.func](*args)
+        else:
+            return self.functions[self.func](args)
+
+
+class EvalCommaSeperator(object):
+    def __init__(self, toks):
+        self.value = toks[0]
+
+    def eval(self):
+        val1 = self.value[0].eval()
+        val2 = self.value[2].eval()
+        if type(val2) is list:
+            val_list = []
+            val_list.append(val1)
+            for val in val2:
+                val_list.append(val)
+            return val_list
+
+        return [val1, val2]
+
+
+class EvalBoolAndOp(object):
+    def __init__(self, toks):
+        self.value = toks[0]
+
+    def eval(self):
+        left = self.value[0].eval()
+        right = self.value[2].eval()
+        return left and right
+
+
+class EvalBoolOrOp(object):
+    def __init__(self, toks):
+        self.value = toks[0]
+
+    def eval(self):
+        left = self.value[0].eval()
+        right = self.value[2].eval()
+        return left or right
+
+_parser = None
+_vars = {}
+
+
+def _def_parser():
+    # Enabling packrat parsing greatly speeds up the parsing.
+    pyparsing.ParserElement.enablePackrat()
+
+    alphas = pyparsing.alphas
+    Combine = pyparsing.Combine
+    Forward = pyparsing.Forward
+    nums = pyparsing.nums
+    oneOf = pyparsing.oneOf
+    opAssoc = pyparsing.opAssoc
+    operatorPrecedence = pyparsing.operatorPrecedence
+    Word = pyparsing.Word
+
+    integer = Word(nums)
+    real = Combine(Word(nums) + '.' + Word(nums))
+    variable = Word(alphas + '_' + '.')
+    number = real | integer
+    expr = Forward()
+    fn = Word(alphas + '_' + '.')
+    operand = number | variable | fn
+
+    signop = oneOf('+ -')
+    addop = oneOf('+ -')
+    multop = oneOf('* /')
+    comparisonop = oneOf(' '.join(EvalComparisonOp.operations.keys()))
+    ternaryop = ('?', ':')
+    boolandop = oneOf('AND and &&')
+    boolorop = oneOf('OR or ||')
+    negateop = oneOf('NOT not !')
+
+    operand.setParseAction(EvalConstant)
+    expr = operatorPrecedence(operand, [
+        (fn, 1, opAssoc.RIGHT, EvalFunction),
+        ("^", 2, opAssoc.RIGHT, EvalPowerOp),
+        (signop, 1, opAssoc.RIGHT, EvalSignOp),
+        (multop, 2, opAssoc.LEFT, EvalMultOp),
+        (addop, 2, opAssoc.LEFT, EvalAddOp),
+        (negateop, 1, opAssoc.RIGHT, EvalNegateOp),
+        (comparisonop, 2, opAssoc.LEFT, EvalComparisonOp),
+        (ternaryop, 3, opAssoc.LEFT, EvalTernaryOp),
+        (boolandop, 2, opAssoc.LEFT, EvalBoolAndOp),
+        (boolorop, 2, opAssoc.LEFT, EvalBoolOrOp),
+        (',', 2, opAssoc.RIGHT, EvalCommaSeperator), ])
+
+    return expr
+
+
+def evaluate(expression, **kwargs):
+    """Evaluates an expression.
+
+    Provides the facility to evaluate mathematical expressions, and to
+    substitute variables from dictionaries into those expressions.
+
+    Supports both integer and floating point values, and automatic
+    promotion where necessary.
+    """
+    global _parser
+    if _parser is None:
+        _parser = _def_parser()
+
+    global _vars
+    _vars = kwargs
+
+    try:
+        result = _parser.parseString(expression, parseAll=True)[0]
+    except pyparsing.ParseException as e:
+        msg = _("ParseException: %s") % six.text_type(e)
+        raise exception.EvaluatorParseException(reason=msg)
+
+    return result.eval()

manila/scheduler/filters/driver.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+from oslo_log import log as logging
+
+from manila.i18n import _LI
+from manila.i18n import _LW
+from manila.scheduler.evaluator import evaluator
+from manila.scheduler.filters import base_host
+from manila.scheduler import utils
+
+
+LOG = logging.getLogger(__name__)
+
+
+class DriverFilter(base_host.BaseHostFilter):
+    """DriverFilter filters hosts based on a 'filter function' and metrics.
+
+    DriverFilter filters based on share host's provided 'filter function'
+    and metrics.
+    """
+
+    def host_passes(self, host_state, filter_properties):
+        """Determines whether a host has a passing filter_function or not."""
+        stats = self._generate_stats(host_state, filter_properties)
+
+        LOG.debug("Driver Filter: Checking host '%s'",
+                  stats['host_stats']['host'])
+        result = self._check_filter_function(stats)
+        LOG.debug("Result: %s", result)
+        LOG.debug("Done checking host '%s'", stats['host_stats']['host'])
+
+        return result
+
+    def _check_filter_function(self, stats):
+        """Checks if a share passes a host's filter function.
+
+           Returns a tuple in the format (filter_passing, filter_invalid).
+           Both values are booleans.
+        """
+        host_stats = stats['host_stats']
+        extra_specs = stats['extra_specs']
+
+        # Check that the share types match
+        if extra_specs is None or 'share_backend_name' not in extra_specs:
+            LOG.warning(_LW("No 'share_backend_name' key in extra_specs. "
+                            "Skipping share backend name check."))
+        elif (extra_specs['share_backend_name'] !=
+                host_stats['share_backend_name']):
+            LOG.warning(_LW("Share backend names do not match: '%(target)s' "
+                            "vs '%(current)s' :: Skipping."),
+                        {'target': extra_specs['share_backend_name'],
+                         'current': host_stats['share_backend_name']})
+            return False
+
+        if stats['filter_function'] is None:
+            LOG.warning(_LW("Filter function not set :: passing host."))
+            return True
+
+        try:
+            filter_result = self._run_evaluator(stats['filter_function'],
+                                                stats)
+        except Exception as ex:
+            # Warn the admin for now that there is an error in the
+            # filter function.
+            LOG.warning(_LW("Error in filtering function "
+                            "'%(function)s' : '%(error)s' :: failing host."),
+                        {'function': stats['filter_function'],
+                         'error': ex, })
+            return False
+
+        msg = _LI("Filter function result for host %(host)s: %(result)s.")
+        args = {'host': stats['host_stats']['host'],
+                'result': six.text_type(filter_result)}
+        LOG.info(msg, args)
+
+        return filter_result
+
+    def _run_evaluator(self, func, stats):
+        """Evaluates a given function using the provided available stats."""
+        host_stats = stats['host_stats']
+        host_caps = stats['host_caps']
+        extra_specs = stats['extra_specs']
+        share_stats = stats['share_stats']
+
+        result = evaluator.evaluate(
+            func,
+            extra=extra_specs,
+            stats=host_stats,
+            capabilities=host_caps,
+            share=share_stats)
+
+        return result
+
+    def _generate_stats(self, host_state, filter_properties):
+        """Generates statistics from host and share data."""
+
+        filter_function = None
+
+        if ('filter_function' in host_state.capabilities and
+                host_state.capabilities['filter_function'] is not None):
+            filter_function = six.text_type(
+                host_state.capabilities['filter_function'])
+
+        stats = utils.generate_stats(host_state, filter_properties)
+
+        stats['filter_function'] = filter_function
+
+        return stats

manila/scheduler/host_manager.py

@@ -46,13 +46,15 @@ host_manager_opts = [
                     'CapacityFilter',
                     'CapabilitiesFilter',
                     'ConsistencyGroupFilter',
+                    'DriverFilter',
                     'ShareReplicationFilter',
                 ],
                 help='Which filter class names to use for filtering hosts '
                      'when not specified in the request.'),
     cfg.ListOpt('scheduler_default_weighers',
                 default=[
-                    'CapacityWeigher'
+                    'CapacityWeigher',
+                    'GoodnessWeigher',
                 ],
                 help='Which weigher class names to use for weighing hosts.')
 ]

manila/scheduler/utils.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+def generate_stats(host_state, properties):
+    """Generates statistics from host and share data."""
+
+    host_stats = {
+        'host': host_state.host,
+        'share_backend_name': host_state.share_backend_name,
+        'vendor_name': host_state.vendor_name,
+        'driver_version': host_state.driver_version,
+        'storage_protocol': host_state.storage_protocol,
+        'qos': host_state.qos,
+        'total_capacity_gb': host_state.total_capacity_gb,
+        'allocated_capacity_gb': host_state.allocated_capacity_gb,
+        'free_capacity_gb': host_state.free_capacity_gb,
+        'reserved_percentage': host_state.reserved_percentage,
+        'driver_handles_share_servers':
+            host_state.driver_handles_share_servers,
+        'thin_provisioning': host_state.thin_provisioning,
+        'updated': host_state.updated,
+        'consistency_group_support': host_state.consistency_group_support,
+        'dedupe': host_state.dedupe,
+        'compression': host_state.compression,
+        'snapshot_support': host_state.snapshot_support,
+        'replication_domain': host_state.replication_domain,
+        'replication_type': host_state.replication_type,
+        'provisioned_capacity_gb': host_state.provisioned_capacity_gb,
+        'pools': host_state.pools,
+        'max_over_subscription_ratio':
+            host_state.max_over_subscription_ratio,
+    }
+
+    host_caps = host_state.capabilities
+
+    share_type = properties.get('share_type', {})
+    extra_specs = share_type.get('extra_specs', {})
+
+    request_spec = properties.get('request_spec', {})
+    share_stats = request_spec.get('resource_properties', {})
+
+    stats = {
+        'host_stats': host_stats,
+        'host_caps': host_caps,
+        'extra_specs': extra_specs,
+        'share_stats': share_stats,
+        'share_type': share_type,
+    }
+
+    return stats

manila/scheduler/weighers/goodness.py

@@ -0,0 +1,125 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log as logging
+import six
+
+from manila.i18n import _LI
+from manila.i18n import _LW
+from manila.scheduler.evaluator import evaluator
+from manila.scheduler import utils
+from manila.scheduler.weighers import base_host
+
+
+LOG = logging.getLogger(__name__)
+
+
+class GoodnessWeigher(base_host.BaseHostWeigher):
+    """Goodness Weigher.  Assign weights based on a host's goodness function.
+
+    Goodness rating is the following:
+
+    .. code-block:: none
+
+          0 -- host is a poor choice
+          .
+          .
+         50 -- host is a good choice
+          .
+          .
+        100 -- host is a perfect choice
+
+    """
+
+    def _weigh_object(self, host_state, weight_properties):
+        """Determine host's goodness rating based on a goodness_function."""
+        stats = self._generate_stats(host_state, weight_properties)
+        LOG.debug("Checking host '%s'", stats['host_stats']['host'])
+        result = self._check_goodness_function(stats)
+        LOG.debug("Goodness: %s", result)
+        LOG.debug("Done checking host '%s'", stats['host_stats']['host'])
+
+        return result
+
+    def _check_goodness_function(self, stats):
+        """Gets a host's goodness rating based on its goodness function."""
+
+        goodness_rating = 0
+
+        if stats['goodness_function'] is None:
+            LOG.warning(_LW("Goodness function not set :: defaulting to "
+                            "minimal goodness rating of 0."))
+        else:
+            try:
+                goodness_result = self._run_evaluator(
+                    stats['goodness_function'],
+                    stats)
+            except Exception as ex:
+                LOG.warning(_LW("Error in goodness_function function "
+                                "'%(function)s' : '%(error)s' :: Defaulting "
+                                "to a goodness of 0."),
+                            {'function': stats['goodness_function'],
+                             'error': ex, })
+                return goodness_rating
+
+            if type(goodness_result) is bool:
+                if goodness_result:
+                    goodness_rating = 100
+            elif goodness_result < 0 or goodness_result > 100:
+                LOG.warning(_LW("Invalid goodness result.  Result must be "
+                                "between 0 and 100.  Result generated: '%s' "
+                                ":: Defaulting to a goodness of 0."),
+                            goodness_result)
+            else:
+                goodness_rating = goodness_result
+
+        msg = _LI("Goodness function result for host %(host)s: %(result)s.")
+        args = {'host': stats['host_stats']['host'],
+                'result': six.text_type(goodness_rating)}
+        LOG.info(msg, args)
+
+        return goodness_rating
+
+    def _run_evaluator(self, func, stats):
+        """Evaluates a given function using the provided available stats."""
+        host_stats = stats['host_stats']
+        host_caps = stats['host_caps']
+        extra_specs = stats['extra_specs']
+        share_stats = stats['share_stats']
+
+        result = evaluator.evaluate(
+            func,
+            extra=extra_specs,
+            stats=host_stats,
+            capabilities=host_caps,
+            share=share_stats)
+
+        return result
+
+    def _generate_stats(self, host_state, weight_properties):
+        """Generates statistics from host and share data."""
+
+        goodness_function = None
+
+        if ('goodness_function' in host_state.capabilities and
+                host_state.capabilities['goodness_function'] is not None):
+            goodness_function = six.text_type(
+                host_state.capabilities['goodness_function'])
+
+        stats = utils.generate_stats(host_state, weight_properties)
+
+        stats['goodness_function'] = goodness_function
+
+        return stats

manila/share/driver.py

@@ -112,6 +112,14 @@ share_opts = [
              "replication between each other. If this option is not "
              "specified in the group, it means that replication is not "
              "enabled on the backend."),
+    cfg.StrOpt('filter_function',
+               default=None,
+               help='String representation for an equation that will be '
+                    'used to filter hosts.'),
+    cfg.StrOpt('goodness_function',
+               default=None,
+               help='String representation for an equation that will be '
+                    'used to determine the goodness of a host.'),
 ]
 
 ssh_opts = [
@@ -808,6 +816,8 @@ class ShareDriver(object):
             pools=self.pools or None,
             snapshot_support=self.snapshots_are_supported,
             replication_domain=self.replication_domain,
+            filter_function=self.get_filter_function(),
+            goodness_function=self.get_goodness_function(),
         )
         if isinstance(data, dict):
             common.update(data)
@@ -1841,3 +1851,57 @@ class ShareDriver(object):
             backend and their status was 'deleting'.
         """
         raise NotImplementedError()
+
+    def get_filter_function(self):
+        """Get filter_function string.
+
+        Returns either the string from the driver instance or global section
+        in manila.conf. If nothing is specified in manila.conf, then try to
+        find the default filter_function. When None is returned the scheduler
+        will always pass the driver instance.
+
+        :return a filter_function string or None
+        """
+        ret_function = self.configuration.filter_function
+        if not ret_function:
+            ret_function = CONF.filter_function
+        if not ret_function:
+            ret_function = self.get_default_filter_function()
+        return ret_function
+
+    def get_goodness_function(self):
+        """Get good_function string.
+
+        Returns either the string from the driver instance or global section
+        in manila.conf. If nothing is specified in manila.conf, then try to
+        find the default goodness_function. When None is returned the scheduler
+        will give the lowest score to the driver instance.
+
+        :return a goodness_function string or None
+        """
+        ret_function = self.configuration.goodness_function
+        if not ret_function:
+            ret_function = CONF.goodness_function
+        if not ret_function:
+            ret_function = self.get_default_goodness_function()
+        return ret_function
+
+    def get_default_filter_function(self):
+        """Get the default filter_function string.
+
+        Each driver could overwrite the method to return a well-known
+        default string if it is available.
+
+        :return: None
+        """
+        return None
+
+    def get_default_goodness_function(self):
+        """Get the default goodness_function string.
+
+        Each driver could overwrite the method to return a well-known
+        default string if it is available.
+
+        :return: None
+        """
+        return None

manila/tests/scheduler/evaluator/test_evaluator.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from manila import exception
+from manila.scheduler.evaluator import evaluator
+from manila import test
+
+
+class EvaluatorTestCase(test.TestCase):
+    def test_simple_integer(self):
+        self.assertEqual(2, evaluator.evaluate("1+1"))
+        self.assertEqual(9, evaluator.evaluate("2+3+4"))
+        self.assertEqual(23, evaluator.evaluate("11+12"))
+        self.assertEqual(30, evaluator.evaluate("5*6"))
+        self.assertEqual(2, evaluator.evaluate("22/11"))
+        self.assertEqual(38, evaluator.evaluate("109-71"))
+        self.assertEqual(
+            493, evaluator.evaluate("872 - 453 + 44 / 22 * 4 + 66"))
+
+    def test_simple_float(self):
+        self.assertEqual(2.0, evaluator.evaluate("1.0 + 1.0"))
+        self.assertEqual(2.5, evaluator.evaluate("1.5 + 1.0"))
+        self.assertEqual(3.0, evaluator.evaluate("1.5 * 2.0"))
+
+    def test_int_float_mix(self):
+        self.assertEqual(2.5, evaluator.evaluate("1.5 + 1"))
+        self.assertEqual(4.25, evaluator.evaluate("8.5 / 2"))
+        self.assertEqual(5.25, evaluator.evaluate("10/4+0.75    + 2"))
+
+    def test_negative_numbers(self):
+        self.assertEqual(-2, evaluator.evaluate("-2"))
+        self.assertEqual(-1, evaluator.evaluate("-2+1"))
+        self.assertEqual(3, evaluator.evaluate("5+-2"))
+
+    def test_exponent(self):
+        self.assertEqual(8, evaluator.evaluate("2^3"))
+        self.assertEqual(-8, evaluator.evaluate("-2 ^ 3"))
+        self.assertEqual(15.625, evaluator.evaluate("2.5 ^ 3"))
+        self.assertEqual(8, evaluator.evaluate("4 ^ 1.5"))
+
+    def test_function(self):
+        self.assertEqual(5, evaluator.evaluate("abs(-5)"))
+        self.assertEqual(2, evaluator.evaluate("abs(2)"))
+        self.assertEqual(1, evaluator.evaluate("min(1, 100)"))
+        self.assertEqual(100, evaluator.evaluate("max(1, 100)"))
+        self.assertEqual(100, evaluator.evaluate("max(1, 2, 100)"))
+
+    def test_parentheses(self):
+        self.assertEqual(1, evaluator.evaluate("(1)"))
+        self.assertEqual(-1, evaluator.evaluate("(-1)"))
+        self.assertEqual(2, evaluator.evaluate("(1+1)"))
+        self.assertEqual(15, evaluator.evaluate("(1+2) * 5"))
+        self.assertEqual(3, evaluator.evaluate("(1+2)*(3-1)/((1+(2-1)))"))
+        self.assertEqual(
+            -8.0, evaluator. evaluate("((1.0 / 0.5) * (2)) *(-2)"))
+
+    def test_comparisons(self):
+        self.assertTrue(evaluator.evaluate("1 < 2"))
+        self.assertTrue(evaluator.evaluate("2 > 1"))
+        self.assertTrue(evaluator.evaluate("2 != 1"))
+        self.assertFalse(evaluator.evaluate("1 > 2"))
+        self.assertFalse(evaluator.evaluate("2 < 1"))
+        self.assertFalse(evaluator.evaluate("2 == 1"))
+        self.assertTrue(evaluator.evaluate("(1 == 1) == !(1 == 2)"))
+
+    def test_logic_ops(self):
+        self.assertTrue(evaluator.evaluate("(1 == 1) AND (2 == 2)"))
+        self.assertTrue(evaluator.evaluate("(1 == 1) and (2 == 2)"))
+        self.assertTrue(evaluator.evaluate("(1 == 1) && (2 == 2)"))
+        self.assertFalse(evaluator.evaluate("(1 == 1) && (5 == 2)"))
+
+        self.assertTrue(evaluator.evaluate("(1 == 1) OR (5 == 2)"))
+        self.assertTrue(evaluator.evaluate("(1 == 1) or (5 == 2)"))
+        self.assertTrue(evaluator.evaluate("(1 == 1) || (5 == 2)"))
+        self.assertFalse(evaluator.evaluate("(5 == 1) || (5 == 2)"))
+
+        self.assertFalse(evaluator.evaluate("(1 == 1) AND NOT (2 == 2)"))
+        self.assertFalse(evaluator.evaluate("(1 == 1) AND not (2 == 2)"))
+        self.assertFalse(evaluator.evaluate("(1 == 1) AND !(2 == 2)"))
+        self.assertTrue(evaluator.evaluate("(1 == 1) AND NOT (5 == 2)"))
+        self.assertTrue(evaluator.evaluate("(1 == 1) OR NOT (2 == 2) "
+                                           "AND (5 == 5)"))
+
+    def test_ternary_conditional(self):
+        self.assertEqual(5, evaluator.evaluate("(1 < 2) ? 5 : 10"))
+        self.assertEqual(10, evaluator.evaluate("(1 > 2) ? 5 : 10"))
+
+    def test_variables_dict(self):
+        stats = {'iops': 1000, 'usage': 0.65, 'count': 503, 'free_space': 407}
+        request = {'iops': 500, 'size': 4}
+        self.assertEqual(1500, evaluator.evaluate("stats.iops + request.iops",
+                                                  stats=stats,
+                                                  request=request))
+
+    def test_missing_var(self):
+        stats = {'iops': 1000, 'usage': 0.65, 'count': 503, 'free_space': 407}
+        request = {'iops': 500, 'size': 4}
+        self.assertRaises(exception.EvaluatorParseException,
+                          evaluator.evaluate,
+                          "foo.bob + 5",
+                          stats=stats, request=request)
+        self.assertRaises(exception.EvaluatorParseException,
+                          evaluator.evaluate,
+                          "stats.bob + 5",
+                          stats=stats, request=request)
+        self.assertRaises(exception.EvaluatorParseException,
+                          evaluator.evaluate,
+                          "fake.var + 1",
+                          stats=stats, request=request, fake=None)
+
+    def test_bad_expression(self):
+        self.assertRaises(exception.EvaluatorParseException,
+                          evaluator.evaluate,
+                          "1/*1")
+
+    def test_nonnumber_comparison(self):
+        nonnumber = {'test': 'foo'}
+        request = {'test': 'bar'}
+        self.assertRaises(
+            exception.EvaluatorParseException,
+            evaluator.evaluate,
+            "nonnumber.test != request.test",
+            nonnumber=nonnumber, request=request)
+
+    def test_div_zero(self):
+        self.assertRaises(exception.EvaluatorParseException,
+                          evaluator.evaluate,
+                          "7 / 0")

manila/tests/scheduler/filters/test_driver.py

@@ -0,0 +1,191 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_context import context
+
+from manila.scheduler.filters import driver
+from manila import test
+from manila.tests.scheduler import fakes
+
+
+class HostFiltersTestCase(test.TestCase):
+
+    def setUp(self):
+        super(HostFiltersTestCase, self).setUp()
+        self.context = context.RequestContext('fake', 'fake')
+        self.filter = driver.DriverFilter()
+
+    def test_passing_function(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'filter_function': '1 == 1',
+                }
+            })
+
+        filter_properties = {'share_type': {}}
+
+        self.assertTrue(self.filter.host_passes(host1, filter_properties))
+
+    def test_failing_function(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'filter_function': '1 == 2',
+                }
+            })
+
+        filter_properties = {'share_type': {}}
+
+        self.assertFalse(self.filter.host_passes(host1, filter_properties))
+
+    def test_no_filter_function(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'filter_function': None,
+                }
+            })
+
+        filter_properties = {'share_type': {}}
+
+        self.assertTrue(self.filter.host_passes(host1, filter_properties))
+
+    def test_not_implemented(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {}
+            })
+
+        filter_properties = {'share_type': {}}
+
+        self.assertTrue(self.filter.host_passes(host1, filter_properties))
+
+    def test_no_share_extra_specs(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'filter_function': '1 == 1',
+                }
+            })
+
+        filter_properties = {'share_type': {}}
+
+        self.assertTrue(self.filter.host_passes(host1, filter_properties))
+
+    def test_extra_specs_wrong_backend(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'filter_function': '1 == 1',
+                }
+            })
+
+        filter_properties = {
+            'share_type': {
+                'extra_specs': {
+                    'share_backend_name': 'foo',
+                }
+            }
+        }
+
+        self.assertFalse(self.filter.host_passes(host1, filter_properties))
+
+    def test_function_extra_spec_replacement(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'filter_function': 'extra.var == 1',
+                }
+            })
+
+        filter_properties = {
+            'share_type': {
+                'extra_specs': {
+                    'var': 1,
+                }
+            }
+        }
+
+        self.assertTrue(self.filter.host_passes(host1, filter_properties))
+
+    def test_function_stats_replacement(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'total_capacity_gb': 100,
+                'capabilities': {
+                    'filter_function': 'stats.total_capacity_gb < 200',
+                }
+            })
+
+        filter_properties = {'share_type': {}}
+
+        self.assertTrue(self.filter.host_passes(host1, filter_properties))
+
+    def test_function_share_replacement(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'filter_function': 'share.size < 5',
+                }
+            })
+
+        filter_properties = {
+            'request_spec': {
+                'resource_properties': {
+                    'size': 1
+                }
+            }
+        }
+
+        self.assertTrue(self.filter.host_passes(host1, filter_properties))
+
+    def test_function_exception_caught(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'filter_function': '1 / 0 == 0',
+                }
+            })
+
+        filter_properties = {}
+
+        self.assertFalse(self.filter.host_passes(host1, filter_properties))
+
+    def test_capabilities(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'foo': 10,
+                    'filter_function': 'capabilities.foo == 10',
+                },
+            })
+
+        filter_properties = {}
+
+        self.assertTrue(self.filter.host_passes(host1, filter_properties))
+
+    def test_wrong_capabilities(self):
+        host1 = fakes.FakeHostState(
+            'host1', {
+                'capabilities': {
+                    'bar': 10,
+                    'filter_function': 'capabilities.foo == 10',
+                },
+            })
+
+        filter_properties = {}
+
+        self.assertFalse(self.filter.host_passes(host1, filter_properties))

manila/tests/scheduler/weighers/test_goodness.py

@@ -0,0 +1,180 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests For Goodness Weigher.
+"""
+
+from manila.scheduler.weighers import goodness
+from manila import test
+from manila.tests.scheduler import fakes
+
+
+class GoodnessWeigherTestCase(test.TestCase):
+
+    def test_goodness_weigher_with_no_goodness_function(self):
+        weigher = goodness.GoodnessWeigher()
+        host_state = fakes.FakeHostState('host1', {
+            'host': 'host.example.com',
+            'capabilities': {
+                'foo': '50'
+            }
+        })
+
+        weight_properties = {}
+        weight = weigher._weigh_object(host_state, weight_properties)
+        self.assertEqual(0, weight)
+
+    def test_goodness_weigher_passing_host(self):
+        weigher = goodness.GoodnessWeigher()
+        host_state = fakes.FakeHostState('host1', {
+            'host': 'host.example.com',
+            'capabilities': {
+                'goodness_function': '100'
+            }
+        })
+        host_state_2 = fakes.FakeHostState('host2', {
+            'host': 'host2.example.com',
+            'capabilities': {
+                'goodness_function': '0'
+            }
+        })
+        host_state_3 = fakes.FakeHostState('host3', {
+            'host': 'host3.example.com',
+            'capabilities': {
+                'goodness_function': '100 / 2'
+            }
+        })
+
+        weight_properties = {}
+        weight = weigher._weigh_object(host_state, weight_properties)
+        self.assertEqual(100, weight)
+        weight = weigher._weigh_object(host_state_2, weight_properties)
+        self.assertEqual(0, weight)
+        weight = weigher._weigh_object(host_state_3, weight_properties)
+        self.assertEqual(50, weight)
+
+    def test_goodness_weigher_capabilities_substitution(self):
+        weigher = goodness.GoodnessWeigher()
+        host_state = fakes.FakeHostState('host1', {
+            'host': 'host.example.com',
+            'capabilities': {
+                'foo': 50,
+                'goodness_function': '10 + capabilities.foo'
+            }
+        })
+
+        weight_properties = {}
+        weight = weigher._weigh_object(host_state, weight_properties)
+        self.assertEqual(60, weight)
+
+    def test_goodness_weigher_extra_specs_substitution(self):
+        weigher = goodness.GoodnessWeigher()
+        host_state = fakes.FakeHostState('host1', {
+            'host': 'host.example.com',
+            'capabilities': {
+                'goodness_function': '10 + extra.foo'
+            }
+        })
+
+        weight_properties = {
+            'share_type': {
+                'extra_specs': {
+                    'foo': 50
+                }
+            }
+        }
+        weight = weigher._weigh_object(host_state, weight_properties)
+        self.assertEqual(60, weight)
+
+    def test_goodness_weigher_share_substitution(self):
+        weigher = goodness.GoodnessWeigher()
+        host_state = fakes.FakeHostState('host1', {
+            'host': 'host.example.com',
+            'capabilities': {
+                'goodness_function': '10 + share.foo'
+            }
+        })
+
+        weight_properties = {
+            'request_spec': {
+                'resource_properties': {
+                    'foo': 50
+                }
+            }
+        }
+        weight = weigher._weigh_object(host_state, weight_properties)
+        self.assertEqual(60, weight)
+
+    def test_goodness_weigher_stats_substitution(self):
+        weigher = goodness.GoodnessWeigher()
+        host_state = fakes.FakeHostState('host1', {
+            'host': 'host.example.com',
+            'capabilities': {
+                'goodness_function': 'stats.free_capacity_gb > 20'
+            },
+            'free_capacity_gb': 50
+        })
+
+        weight_properties = {}
+        weight = weigher._weigh_object(host_state, weight_properties)
+        self.assertEqual(100, weight)
+
+    def test_goodness_weigher_invalid_substitution(self):
+        weigher = goodness.GoodnessWeigher()
+        host_state = fakes.FakeHostState('host1', {
+            'host': 'host.example.com',
+            'capabilities': {
+                'goodness_function': '10 + stats.my_val'
+            },
+            'foo': 50
+        })
+
+        weight_properties = {}
+        weight = weigher._weigh_object(host_state, weight_properties)
+        self.assertEqual(0, weight)
+
+    def test_goodness_weigher_host_rating_out_of_bounds(self):
+        weigher = goodness.GoodnessWeigher()
+        host_state = fakes.FakeHostState('host1', {
+            'host': 'host.example.com',
+            'capabilities': {
+                'goodness_function': '-10'
+            }
+        })
+        host_state_2 = fakes.FakeHostState('host2', {
+            'host': 'host2.example.com',
+            'capabilities': {
+                'goodness_function': '200'
+            }
+        })
+
+        weight_properties = {}
+        weight = weigher._weigh_object(host_state, weight_properties)
+        self.assertEqual(0, weight)
+        weight = weigher._weigh_object(host_state_2, weight_properties)
+        self.assertEqual(0, weight)
+
+    def test_goodness_weigher_invalid_goodness_function(self):
+        weigher = goodness.GoodnessWeigher()
+        host_state = fakes.FakeHostState('host1', {
+            'host': 'host.example.com',
+            'capabilities': {
+                'goodness_function': '50 / 0'
+            }
+        })
+
+        weight_properties = {}
+        weight = weigher._weigh_object(host_state, weight_properties)
+        self.assertEqual(0, weight)

manila/tests/share/drivers/emc/test_driver.py

@@ -125,6 +125,8 @@ class EMCShareFrameworkTestCase(test.TestCase):
         data['pools'] = None
         data['snapshot_support'] = True
         data['replication_domain'] = None
+        data['filter_function'] = None
+        data['goodness_function'] = None
         self.assertEqual(data, self.driver._stats)
 
     def _fake_safe_get(self, value):

manila/tests/share/drivers/glusterfs/test_glusterfs_native.py

@@ -257,6 +257,8 @@ class GlusterfsNativeShareDriverTestCase(test.TestCase):
             'pools': None,
             'snapshot_support': True,
             'replication_domain': None,
+            'filter_function': None,
+            'goodness_function': None,
         }
         self.assertEqual(test_data, self._driver._stats)
 

manila/tests/share/drivers/hpe/test_hpe_3par_driver.py

@@ -58,6 +58,8 @@ class HPE3ParDriverTestCase(test.TestCase):
         self.conf.network_config_group = 'test_network_config_group'
         self.conf.admin_network_config_group = (
             'test_admin_network_config_group')
+        self.conf.filter_function = None
+        self.conf.goodness_function = None
 
         def safe_get(attr):
             try:
@@ -560,6 +562,8 @@ class HPE3ParDriverTestCase(test.TestCase):
             'vendor_name': 'HPE',
             'pools': None,
             'replication_domain': None,
+            'filter_function': None,
+            'goodness_function': None,
         }
 
         result = self.driver.get_share_stats(refresh=True)
@@ -618,6 +622,8 @@ class HPE3ParDriverTestCase(test.TestCase):
             'hp3par_flash_cache': False,
             'snapshot_support': True,
             'replication_domain': None,
+            'filter_function': None,
+            'goodness_function': None,
         }
 
         result = self.driver.get_share_stats(refresh=True)
@@ -652,6 +658,8 @@ class HPE3ParDriverTestCase(test.TestCase):
             'vendor_name': 'HPE',
             'snapshot_support': True,
             'replication_domain': None,
+            'filter_function': None,
+            'goodness_function': None,
         }
 
         result = self.driver.get_share_stats(refresh=True)

manila/tests/share/drivers/huawei/test_huawei_nas.py

@@ -746,6 +746,8 @@ class HuaweiShareDriverTestCase(test.TestCase):
         self.configuration.max_over_subscription_ratio = 1
         self.configuration.driver_handles_share_servers = False
         self.configuration.replication_domain = None
+        self.configuration.filter_function = None
+        self.configuration.goodness_function = None
 
         self.tmp_dir = tempfile.mkdtemp()
         self.fake_conf_file = self.tmp_dir + '/manila_huawei_conf.xml'
@@ -2189,6 +2191,8 @@ class HuaweiShareDriverTestCase(test.TestCase):
         expected['qos'] = True
         expected["snapshot_support"] = True
         expected['replication_domain'] = None
+        expected['filter_function'] = None
+        expected['goodness_function'] = None
         expected["pools"] = []
         pool = dict(
             pool_name='OpenStack_Pool',

manila/tests/share/drivers/zfsonlinux/test_driver.py

@@ -59,6 +59,8 @@ class FakeConfig(object):
             "reserved_share_percentage", 0)
         self.max_over_subscription_ratio = kwargs.get(
             "max_over_subscription_ratio", 15.0)
+        self.filter_function = kwargs.get("filter_function", None)
+        self.goodness_function = kwargs.get("goodness_function", None)
 
     def safe_get(self, key):
         return getattr(self, key)
@@ -303,6 +305,8 @@ class ZFSonLinuxShareDriverTestCase(test.TestCase):
             'storage_protocol': 'NFS',
             'total_capacity_gb': 'unknown',
             'vendor_name': 'Open Source',
+            'filter_function': None,
+            'goodness_function': None,
         }
         if replication_domain:
             expected['replication_type'] = 'readable'

releasenotes/notes/driver-filter-91e2c60c9d1a48dd.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - Add DriverFilter and GoodnessWeigher to manila's scheduler.
+    These can use two new properties provided by backends, 'filter_function'
+    and 'goodness_function', which can be used to filter and weigh qualified
+    backends, respectively.
+upgrade:
+  - To add DriverFilter and GoodnessWeigher to an active deployment, their
+    references must be added to the filters and weighers sections on
+    entry_points.txt.

requirements.txt

@@ -27,6 +27,7 @@ oslo.concurrency>=3.8.0 # Apache-2.0
 paramiko>=2.0 # LGPLv2.1+
 Paste # MIT
 PasteDeploy>=1.5.0 # MIT
+pyparsing>=2.0.1 # MIT
 python-neutronclient>=4.2.0 # Apache-2.0
 keystoneauth1>=2.7.0 # Apache-2.0
 keystonemiddleware!=4.1.0,!=4.5.0,>=4.0.0 # Apache-2.0

setup.cfg

@@ -38,12 +38,14 @@ manila.scheduler.filters =
     CapabilitiesFilter = manila.scheduler.filters.capabilities:CapabilitiesFilter
     CapacityFilter = manila.scheduler.filters.capacity:CapacityFilter
     ConsistencyGroupFilter = manila.scheduler.filters.consistency_group:ConsistencyGroupFilter
+    DriverFilter = manila.scheduler.filters.driver:DriverFilter
     IgnoreAttemptedHostsFilter = manila.scheduler.filters.ignore_attempted_hosts:IgnoreAttemptedHostsFilter
     JsonFilter = manila.scheduler.filters.json:JsonFilter
     RetryFilter = manila.scheduler.filters.retry:RetryFilter
     ShareReplicationFilter = manila.scheduler.filters.share_replication:ShareReplicationFilter
 manila.scheduler.weighers =
     CapacityWeigher = manila.scheduler.weighers.capacity:CapacityWeigher
+    GoodnessWeigher = manila.scheduler.weighers.goodness:GoodnessWeigher
     PoolWeigher = manila.scheduler.weighers.pool:PoolWeigher
 # These are for backwards compat with Havana notification_driver configuration values
 oslo_messaging.notify.drivers =