From f10776d832b03ff4b624d6b5919951306db332bd Mon Sep 17 00:00:00 2001 From: tpsilva Date: Thu, 30 Jun 2016 14:46:50 -0300 Subject: [PATCH] Add DriverFilter and GoodnessWeigher to manila This patch ports cinder's DriverFilter and GoodnessWeigher to manila. These can use two new properties provided by backends, 'filter_function' and 'goodness_function', which can be used to filter and weigh qualified backends, respectively. Reference for cinder spec: I59b607a88953a346aa35e67e785a0417a7ce8cc9 Reference for cinder commit: I38408ab49b6ed869c1faae746ee64a3bae86be58 DocImpact Change-Id: I873f4152e16efdeb30ceae26335a7974dc9b4b69 Implements: blueprint driver-filter-goodness-weigher --- manila/exception.py | 4 + manila/scheduler/evaluator/__init__.py | 0 manila/scheduler/evaluator/evaluator.py | 297 ++++++++++++++++++ manila/scheduler/filters/driver.py | 123 ++++++++ manila/scheduler/host_manager.py | 4 +- manila/scheduler/utils.py | 63 ++++ manila/scheduler/weighers/goodness.py | 125 ++++++++ manila/share/driver.py | 64 ++++ manila/tests/scheduler/evaluator/__init__.py | 0 .../scheduler/evaluator/test_evaluator.py | 140 +++++++++ manila/tests/scheduler/filters/test_driver.py | 191 +++++++++++ .../tests/scheduler/weighers/test_goodness.py | 180 +++++++++++ manila/tests/share/drivers/emc/test_driver.py | 2 + .../glusterfs/test_glusterfs_native.py | 2 + .../share/drivers/hpe/test_hpe_3par_driver.py | 8 + .../share/drivers/huawei/test_huawei_nas.py | 4 + .../share/drivers/zfsonlinux/test_driver.py | 4 + .../notes/driver-filter-91e2c60c9d1a48dd.yaml | 10 + requirements.txt | 1 + setup.cfg | 2 + 20 files changed, 1223 insertions(+), 1 deletion(-) create mode 100644 manila/scheduler/evaluator/__init__.py create mode 100644 manila/scheduler/evaluator/evaluator.py create mode 100644 manila/scheduler/filters/driver.py create mode 100644 manila/scheduler/utils.py create mode 100644 manila/scheduler/weighers/goodness.py create mode 100644 manila/tests/scheduler/evaluator/__init__.py create mode 100644 manila/tests/scheduler/evaluator/test_evaluator.py create mode 100644 manila/tests/scheduler/filters/test_driver.py create mode 100644 manila/tests/scheduler/weighers/test_goodness.py create mode 100644 releasenotes/notes/driver-filter-91e2c60c9d1a48dd.yaml diff --git a/manila/exception.py b/manila/exception.py index 1926a417e8..a65b009027 100644 --- a/manila/exception.py +++ b/manila/exception.py @@ -782,3 +782,7 @@ class TegileAPIException(ShareBackendException): class StorageCommunicationException(ShareBackendException): message = _("Could not communicate with storage array.") + + +class EvaluatorParseException(ManilaException): + message = _("Error during evaluator parsing: %(reason)s") diff --git a/manila/scheduler/evaluator/__init__.py b/manila/scheduler/evaluator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/scheduler/evaluator/evaluator.py b/manila/scheduler/evaluator/evaluator.py new file mode 100644 index 0000000000..7c72d8be90 --- /dev/null +++ b/manila/scheduler/evaluator/evaluator.py @@ -0,0 +1,297 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import operator +import re + +import pyparsing +import six + +from manila import exception +from manila.i18n import _ + + +def _operatorOperands(tokenList): + it = iter(tokenList) + while 1: + try: + op1 = next(it) + op2 = next(it) + yield(op1, op2) + except StopIteration: + break + + +class EvalConstant(object): + def __init__(self, toks): + self.value = toks[0] + + def eval(self): + result = self.value + if (isinstance(result, six.string_types) and + re.match("^[a-zA-Z_]+\.[a-zA-Z_]+$", result)): + (which_dict, entry) = result.split('.') + try: + result = _vars[which_dict][entry] + except KeyError as e: + msg = _("KeyError: %s") % six.text_type(e) + raise exception.EvaluatorParseException(reason=msg) + except TypeError as e: + msg = _("TypeError: %s") % six.text_type(e) + raise exception.EvaluatorParseException(reason=msg) + + try: + result = int(result) + except ValueError: + try: + result = float(result) + except ValueError as e: + msg = _("ValueError: %s") % six.text_type(e) + raise exception.EvaluatorParseException(reason=msg) + + return result + + +class EvalSignOp(object): + operations = { + '+': 1, + '-': -1, + } + + def __init__(self, toks): + self.sign, self.value = toks[0] + + def eval(self): + return self.operations[self.sign] * self.value.eval() + + +class EvalAddOp(object): + def __init__(self, toks): + self.value = toks[0] + + def eval(self): + sum = self.value[0].eval() + for op, val in _operatorOperands(self.value[1:]): + if op == '+': + sum += val.eval() + elif op == '-': + sum -= val.eval() + return sum + + +class EvalMultOp(object): + def __init__(self, toks): + self.value = toks[0] + + def eval(self): + prod = self.value[0].eval() + for op, val in _operatorOperands(self.value[1:]): + try: + if op == '*': + prod *= val.eval() + elif op == '/': + prod /= float(val.eval()) + except ZeroDivisionError as e: + msg = _("ZeroDivisionError: %s") % six.text_type(e) + raise exception.EvaluatorParseException(reason=msg) + return prod + + +class EvalPowerOp(object): + def __init__(self, toks): + self.value = toks[0] + + def eval(self): + prod = self.value[0].eval() + for op, val in _operatorOperands(self.value[1:]): + prod = pow(prod, val.eval()) + return prod + + +class EvalNegateOp(object): + def __init__(self, toks): + self.negation, self.value = toks[0] + + def eval(self): + return not self.value.eval() + + +class EvalComparisonOp(object): + operations = { + "<": operator.lt, + "<=": operator.le, + ">": operator.gt, + ">=": operator.ge, + "!=": operator.ne, + "==": operator.eq, + "<>": operator.ne, + } + + def __init__(self, toks): + self.value = toks[0] + + def eval(self): + val1 = self.value[0].eval() + for op, val in _operatorOperands(self.value[1:]): + fn = self.operations[op] + val2 = val.eval() + if not fn(val1, val2): + break + val1 = val2 + else: + return True + return False + + +class EvalTernaryOp(object): + def __init__(self, toks): + self.value = toks[0] + + def eval(self): + condition = self.value[0].eval() + if condition: + return self.value[2].eval() + else: + return self.value[4].eval() + + +class EvalFunction(object): + functions = { + "abs": abs, + "max": max, + "min": min, + } + + def __init__(self, toks): + self.func, self.value = toks[0] + + def eval(self): + args = self.value.eval() + if type(args) is list: + return self.functions[self.func](*args) + else: + return self.functions[self.func](args) + + +class EvalCommaSeperator(object): + def __init__(self, toks): + self.value = toks[0] + + def eval(self): + val1 = self.value[0].eval() + val2 = self.value[2].eval() + if type(val2) is list: + val_list = [] + val_list.append(val1) + for val in val2: + val_list.append(val) + return val_list + + return [val1, val2] + + +class EvalBoolAndOp(object): + def __init__(self, toks): + self.value = toks[0] + + def eval(self): + left = self.value[0].eval() + right = self.value[2].eval() + return left and right + + +class EvalBoolOrOp(object): + def __init__(self, toks): + self.value = toks[0] + + def eval(self): + left = self.value[0].eval() + right = self.value[2].eval() + return left or right + +_parser = None +_vars = {} + + +def _def_parser(): + # Enabling packrat parsing greatly speeds up the parsing. + pyparsing.ParserElement.enablePackrat() + + alphas = pyparsing.alphas + Combine = pyparsing.Combine + Forward = pyparsing.Forward + nums = pyparsing.nums + oneOf = pyparsing.oneOf + opAssoc = pyparsing.opAssoc + operatorPrecedence = pyparsing.operatorPrecedence + Word = pyparsing.Word + + integer = Word(nums) + real = Combine(Word(nums) + '.' + Word(nums)) + variable = Word(alphas + '_' + '.') + number = real | integer + expr = Forward() + fn = Word(alphas + '_' + '.') + operand = number | variable | fn + + signop = oneOf('+ -') + addop = oneOf('+ -') + multop = oneOf('* /') + comparisonop = oneOf(' '.join(EvalComparisonOp.operations.keys())) + ternaryop = ('?', ':') + boolandop = oneOf('AND and &&') + boolorop = oneOf('OR or ||') + negateop = oneOf('NOT not !') + + operand.setParseAction(EvalConstant) + expr = operatorPrecedence(operand, [ + (fn, 1, opAssoc.RIGHT, EvalFunction), + ("^", 2, opAssoc.RIGHT, EvalPowerOp), + (signop, 1, opAssoc.RIGHT, EvalSignOp), + (multop, 2, opAssoc.LEFT, EvalMultOp), + (addop, 2, opAssoc.LEFT, EvalAddOp), + (negateop, 1, opAssoc.RIGHT, EvalNegateOp), + (comparisonop, 2, opAssoc.LEFT, EvalComparisonOp), + (ternaryop, 3, opAssoc.LEFT, EvalTernaryOp), + (boolandop, 2, opAssoc.LEFT, EvalBoolAndOp), + (boolorop, 2, opAssoc.LEFT, EvalBoolOrOp), + (',', 2, opAssoc.RIGHT, EvalCommaSeperator), ]) + + return expr + + +def evaluate(expression, **kwargs): + """Evaluates an expression. + + Provides the facility to evaluate mathematical expressions, and to + substitute variables from dictionaries into those expressions. + + Supports both integer and floating point values, and automatic + promotion where necessary. + """ + global _parser + if _parser is None: + _parser = _def_parser() + + global _vars + _vars = kwargs + + try: + result = _parser.parseString(expression, parseAll=True)[0] + except pyparsing.ParseException as e: + msg = _("ParseException: %s") % six.text_type(e) + raise exception.EvaluatorParseException(reason=msg) + + return result.eval() diff --git a/manila/scheduler/filters/driver.py b/manila/scheduler/filters/driver.py new file mode 100644 index 0000000000..681635fd9d --- /dev/null +++ b/manila/scheduler/filters/driver.py @@ -0,0 +1,123 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six + +from oslo_log import log as logging + +from manila.i18n import _LI +from manila.i18n import _LW +from manila.scheduler.evaluator import evaluator +from manila.scheduler.filters import base_host +from manila.scheduler import utils + + +LOG = logging.getLogger(__name__) + + +class DriverFilter(base_host.BaseHostFilter): + """DriverFilter filters hosts based on a 'filter function' and metrics. + + DriverFilter filters based on share host's provided 'filter function' + and metrics. + """ + + def host_passes(self, host_state, filter_properties): + """Determines whether a host has a passing filter_function or not.""" + stats = self._generate_stats(host_state, filter_properties) + + LOG.debug("Driver Filter: Checking host '%s'", + stats['host_stats']['host']) + result = self._check_filter_function(stats) + LOG.debug("Result: %s", result) + LOG.debug("Done checking host '%s'", stats['host_stats']['host']) + + return result + + def _check_filter_function(self, stats): + """Checks if a share passes a host's filter function. + + Returns a tuple in the format (filter_passing, filter_invalid). + Both values are booleans. + """ + host_stats = stats['host_stats'] + extra_specs = stats['extra_specs'] + + # Check that the share types match + if extra_specs is None or 'share_backend_name' not in extra_specs: + LOG.warning(_LW("No 'share_backend_name' key in extra_specs. " + "Skipping share backend name check.")) + elif (extra_specs['share_backend_name'] != + host_stats['share_backend_name']): + LOG.warning(_LW("Share backend names do not match: '%(target)s' " + "vs '%(current)s' :: Skipping."), + {'target': extra_specs['share_backend_name'], + 'current': host_stats['share_backend_name']}) + return False + + if stats['filter_function'] is None: + LOG.warning(_LW("Filter function not set :: passing host.")) + return True + + try: + filter_result = self._run_evaluator(stats['filter_function'], + stats) + except Exception as ex: + # Warn the admin for now that there is an error in the + # filter function. + LOG.warning(_LW("Error in filtering function " + "'%(function)s' : '%(error)s' :: failing host."), + {'function': stats['filter_function'], + 'error': ex, }) + return False + + msg = _LI("Filter function result for host %(host)s: %(result)s.") + args = {'host': stats['host_stats']['host'], + 'result': six.text_type(filter_result)} + LOG.info(msg, args) + + return filter_result + + def _run_evaluator(self, func, stats): + """Evaluates a given function using the provided available stats.""" + host_stats = stats['host_stats'] + host_caps = stats['host_caps'] + extra_specs = stats['extra_specs'] + share_stats = stats['share_stats'] + + result = evaluator.evaluate( + func, + extra=extra_specs, + stats=host_stats, + capabilities=host_caps, + share=share_stats) + + return result + + def _generate_stats(self, host_state, filter_properties): + """Generates statistics from host and share data.""" + + filter_function = None + + if ('filter_function' in host_state.capabilities and + host_state.capabilities['filter_function'] is not None): + filter_function = six.text_type( + host_state.capabilities['filter_function']) + + stats = utils.generate_stats(host_state, filter_properties) + + stats['filter_function'] = filter_function + + return stats diff --git a/manila/scheduler/host_manager.py b/manila/scheduler/host_manager.py index 9473c51d4d..237ecf2d18 100644 --- a/manila/scheduler/host_manager.py +++ b/manila/scheduler/host_manager.py @@ -46,13 +46,15 @@ host_manager_opts = [ 'CapacityFilter', 'CapabilitiesFilter', 'ConsistencyGroupFilter', + 'DriverFilter', 'ShareReplicationFilter', ], help='Which filter class names to use for filtering hosts ' 'when not specified in the request.'), cfg.ListOpt('scheduler_default_weighers', default=[ - 'CapacityWeigher' + 'CapacityWeigher', + 'GoodnessWeigher', ], help='Which weigher class names to use for weighing hosts.') ] diff --git a/manila/scheduler/utils.py b/manila/scheduler/utils.py new file mode 100644 index 0000000000..2a24fcc1dd --- /dev/null +++ b/manila/scheduler/utils.py @@ -0,0 +1,63 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def generate_stats(host_state, properties): + """Generates statistics from host and share data.""" + + host_stats = { + 'host': host_state.host, + 'share_backend_name': host_state.share_backend_name, + 'vendor_name': host_state.vendor_name, + 'driver_version': host_state.driver_version, + 'storage_protocol': host_state.storage_protocol, + 'qos': host_state.qos, + 'total_capacity_gb': host_state.total_capacity_gb, + 'allocated_capacity_gb': host_state.allocated_capacity_gb, + 'free_capacity_gb': host_state.free_capacity_gb, + 'reserved_percentage': host_state.reserved_percentage, + 'driver_handles_share_servers': + host_state.driver_handles_share_servers, + 'thin_provisioning': host_state.thin_provisioning, + 'updated': host_state.updated, + 'consistency_group_support': host_state.consistency_group_support, + 'dedupe': host_state.dedupe, + 'compression': host_state.compression, + 'snapshot_support': host_state.snapshot_support, + 'replication_domain': host_state.replication_domain, + 'replication_type': host_state.replication_type, + 'provisioned_capacity_gb': host_state.provisioned_capacity_gb, + 'pools': host_state.pools, + 'max_over_subscription_ratio': + host_state.max_over_subscription_ratio, + } + + host_caps = host_state.capabilities + + share_type = properties.get('share_type', {}) + extra_specs = share_type.get('extra_specs', {}) + + request_spec = properties.get('request_spec', {}) + share_stats = request_spec.get('resource_properties', {}) + + stats = { + 'host_stats': host_stats, + 'host_caps': host_caps, + 'extra_specs': extra_specs, + 'share_stats': share_stats, + 'share_type': share_type, + } + + return stats diff --git a/manila/scheduler/weighers/goodness.py b/manila/scheduler/weighers/goodness.py new file mode 100644 index 0000000000..45b23c979d --- /dev/null +++ b/manila/scheduler/weighers/goodness.py @@ -0,0 +1,125 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +import six + +from manila.i18n import _LI +from manila.i18n import _LW +from manila.scheduler.evaluator import evaluator +from manila.scheduler import utils +from manila.scheduler.weighers import base_host + + +LOG = logging.getLogger(__name__) + + +class GoodnessWeigher(base_host.BaseHostWeigher): + """Goodness Weigher. Assign weights based on a host's goodness function. + + Goodness rating is the following: + + .. code-block:: none + + 0 -- host is a poor choice + . + . + 50 -- host is a good choice + . + . + 100 -- host is a perfect choice + + """ + + def _weigh_object(self, host_state, weight_properties): + """Determine host's goodness rating based on a goodness_function.""" + stats = self._generate_stats(host_state, weight_properties) + LOG.debug("Checking host '%s'", stats['host_stats']['host']) + result = self._check_goodness_function(stats) + LOG.debug("Goodness: %s", result) + LOG.debug("Done checking host '%s'", stats['host_stats']['host']) + + return result + + def _check_goodness_function(self, stats): + """Gets a host's goodness rating based on its goodness function.""" + + goodness_rating = 0 + + if stats['goodness_function'] is None: + LOG.warning(_LW("Goodness function not set :: defaulting to " + "minimal goodness rating of 0.")) + else: + try: + goodness_result = self._run_evaluator( + stats['goodness_function'], + stats) + except Exception as ex: + LOG.warning(_LW("Error in goodness_function function " + "'%(function)s' : '%(error)s' :: Defaulting " + "to a goodness of 0."), + {'function': stats['goodness_function'], + 'error': ex, }) + return goodness_rating + + if type(goodness_result) is bool: + if goodness_result: + goodness_rating = 100 + elif goodness_result < 0 or goodness_result > 100: + LOG.warning(_LW("Invalid goodness result. Result must be " + "between 0 and 100. Result generated: '%s' " + ":: Defaulting to a goodness of 0."), + goodness_result) + else: + goodness_rating = goodness_result + + msg = _LI("Goodness function result for host %(host)s: %(result)s.") + args = {'host': stats['host_stats']['host'], + 'result': six.text_type(goodness_rating)} + LOG.info(msg, args) + + return goodness_rating + + def _run_evaluator(self, func, stats): + """Evaluates a given function using the provided available stats.""" + host_stats = stats['host_stats'] + host_caps = stats['host_caps'] + extra_specs = stats['extra_specs'] + share_stats = stats['share_stats'] + + result = evaluator.evaluate( + func, + extra=extra_specs, + stats=host_stats, + capabilities=host_caps, + share=share_stats) + + return result + + def _generate_stats(self, host_state, weight_properties): + """Generates statistics from host and share data.""" + + goodness_function = None + + if ('goodness_function' in host_state.capabilities and + host_state.capabilities['goodness_function'] is not None): + goodness_function = six.text_type( + host_state.capabilities['goodness_function']) + + stats = utils.generate_stats(host_state, weight_properties) + + stats['goodness_function'] = goodness_function + + return stats diff --git a/manila/share/driver.py b/manila/share/driver.py index 5b6b099219..4d1237b56d 100644 --- a/manila/share/driver.py +++ b/manila/share/driver.py @@ -112,6 +112,14 @@ share_opts = [ "replication between each other. If this option is not " "specified in the group, it means that replication is not " "enabled on the backend."), + cfg.StrOpt('filter_function', + default=None, + help='String representation for an equation that will be ' + 'used to filter hosts.'), + cfg.StrOpt('goodness_function', + default=None, + help='String representation for an equation that will be ' + 'used to determine the goodness of a host.'), ] ssh_opts = [ @@ -808,6 +816,8 @@ class ShareDriver(object): pools=self.pools or None, snapshot_support=self.snapshots_are_supported, replication_domain=self.replication_domain, + filter_function=self.get_filter_function(), + goodness_function=self.get_goodness_function(), ) if isinstance(data, dict): common.update(data) @@ -1841,3 +1851,57 @@ class ShareDriver(object): backend and their status was 'deleting'. """ raise NotImplementedError() + + def get_filter_function(self): + """Get filter_function string. + + Returns either the string from the driver instance or global section + in manila.conf. If nothing is specified in manila.conf, then try to + find the default filter_function. When None is returned the scheduler + will always pass the driver instance. + + :return a filter_function string or None + """ + ret_function = self.configuration.filter_function + if not ret_function: + ret_function = CONF.filter_function + if not ret_function: + ret_function = self.get_default_filter_function() + return ret_function + + def get_goodness_function(self): + """Get good_function string. + + Returns either the string from the driver instance or global section + in manila.conf. If nothing is specified in manila.conf, then try to + find the default goodness_function. When None is returned the scheduler + will give the lowest score to the driver instance. + + :return a goodness_function string or None + """ + ret_function = self.configuration.goodness_function + if not ret_function: + ret_function = CONF.goodness_function + if not ret_function: + ret_function = self.get_default_goodness_function() + return ret_function + + def get_default_filter_function(self): + """Get the default filter_function string. + + Each driver could overwrite the method to return a well-known + default string if it is available. + + :return: None + """ + return None + + def get_default_goodness_function(self): + """Get the default goodness_function string. + + Each driver could overwrite the method to return a well-known + default string if it is available. + + :return: None + """ + return None diff --git a/manila/tests/scheduler/evaluator/__init__.py b/manila/tests/scheduler/evaluator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/tests/scheduler/evaluator/test_evaluator.py b/manila/tests/scheduler/evaluator/test_evaluator.py new file mode 100644 index 0000000000..44c5a47c23 --- /dev/null +++ b/manila/tests/scheduler/evaluator/test_evaluator.py @@ -0,0 +1,140 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from manila import exception +from manila.scheduler.evaluator import evaluator +from manila import test + + +class EvaluatorTestCase(test.TestCase): + def test_simple_integer(self): + self.assertEqual(2, evaluator.evaluate("1+1")) + self.assertEqual(9, evaluator.evaluate("2+3+4")) + self.assertEqual(23, evaluator.evaluate("11+12")) + self.assertEqual(30, evaluator.evaluate("5*6")) + self.assertEqual(2, evaluator.evaluate("22/11")) + self.assertEqual(38, evaluator.evaluate("109-71")) + self.assertEqual( + 493, evaluator.evaluate("872 - 453 + 44 / 22 * 4 + 66")) + + def test_simple_float(self): + self.assertEqual(2.0, evaluator.evaluate("1.0 + 1.0")) + self.assertEqual(2.5, evaluator.evaluate("1.5 + 1.0")) + self.assertEqual(3.0, evaluator.evaluate("1.5 * 2.0")) + + def test_int_float_mix(self): + self.assertEqual(2.5, evaluator.evaluate("1.5 + 1")) + self.assertEqual(4.25, evaluator.evaluate("8.5 / 2")) + self.assertEqual(5.25, evaluator.evaluate("10/4+0.75 + 2")) + + def test_negative_numbers(self): + self.assertEqual(-2, evaluator.evaluate("-2")) + self.assertEqual(-1, evaluator.evaluate("-2+1")) + self.assertEqual(3, evaluator.evaluate("5+-2")) + + def test_exponent(self): + self.assertEqual(8, evaluator.evaluate("2^3")) + self.assertEqual(-8, evaluator.evaluate("-2 ^ 3")) + self.assertEqual(15.625, evaluator.evaluate("2.5 ^ 3")) + self.assertEqual(8, evaluator.evaluate("4 ^ 1.5")) + + def test_function(self): + self.assertEqual(5, evaluator.evaluate("abs(-5)")) + self.assertEqual(2, evaluator.evaluate("abs(2)")) + self.assertEqual(1, evaluator.evaluate("min(1, 100)")) + self.assertEqual(100, evaluator.evaluate("max(1, 100)")) + self.assertEqual(100, evaluator.evaluate("max(1, 2, 100)")) + + def test_parentheses(self): + self.assertEqual(1, evaluator.evaluate("(1)")) + self.assertEqual(-1, evaluator.evaluate("(-1)")) + self.assertEqual(2, evaluator.evaluate("(1+1)")) + self.assertEqual(15, evaluator.evaluate("(1+2) * 5")) + self.assertEqual(3, evaluator.evaluate("(1+2)*(3-1)/((1+(2-1)))")) + self.assertEqual( + -8.0, evaluator. evaluate("((1.0 / 0.5) * (2)) *(-2)")) + + def test_comparisons(self): + self.assertTrue(evaluator.evaluate("1 < 2")) + self.assertTrue(evaluator.evaluate("2 > 1")) + self.assertTrue(evaluator.evaluate("2 != 1")) + self.assertFalse(evaluator.evaluate("1 > 2")) + self.assertFalse(evaluator.evaluate("2 < 1")) + self.assertFalse(evaluator.evaluate("2 == 1")) + self.assertTrue(evaluator.evaluate("(1 == 1) == !(1 == 2)")) + + def test_logic_ops(self): + self.assertTrue(evaluator.evaluate("(1 == 1) AND (2 == 2)")) + self.assertTrue(evaluator.evaluate("(1 == 1) and (2 == 2)")) + self.assertTrue(evaluator.evaluate("(1 == 1) && (2 == 2)")) + self.assertFalse(evaluator.evaluate("(1 == 1) && (5 == 2)")) + + self.assertTrue(evaluator.evaluate("(1 == 1) OR (5 == 2)")) + self.assertTrue(evaluator.evaluate("(1 == 1) or (5 == 2)")) + self.assertTrue(evaluator.evaluate("(1 == 1) || (5 == 2)")) + self.assertFalse(evaluator.evaluate("(5 == 1) || (5 == 2)")) + + self.assertFalse(evaluator.evaluate("(1 == 1) AND NOT (2 == 2)")) + self.assertFalse(evaluator.evaluate("(1 == 1) AND not (2 == 2)")) + self.assertFalse(evaluator.evaluate("(1 == 1) AND !(2 == 2)")) + self.assertTrue(evaluator.evaluate("(1 == 1) AND NOT (5 == 2)")) + self.assertTrue(evaluator.evaluate("(1 == 1) OR NOT (2 == 2) " + "AND (5 == 5)")) + + def test_ternary_conditional(self): + self.assertEqual(5, evaluator.evaluate("(1 < 2) ? 5 : 10")) + self.assertEqual(10, evaluator.evaluate("(1 > 2) ? 5 : 10")) + + def test_variables_dict(self): + stats = {'iops': 1000, 'usage': 0.65, 'count': 503, 'free_space': 407} + request = {'iops': 500, 'size': 4} + self.assertEqual(1500, evaluator.evaluate("stats.iops + request.iops", + stats=stats, + request=request)) + + def test_missing_var(self): + stats = {'iops': 1000, 'usage': 0.65, 'count': 503, 'free_space': 407} + request = {'iops': 500, 'size': 4} + self.assertRaises(exception.EvaluatorParseException, + evaluator.evaluate, + "foo.bob + 5", + stats=stats, request=request) + self.assertRaises(exception.EvaluatorParseException, + evaluator.evaluate, + "stats.bob + 5", + stats=stats, request=request) + self.assertRaises(exception.EvaluatorParseException, + evaluator.evaluate, + "fake.var + 1", + stats=stats, request=request, fake=None) + + def test_bad_expression(self): + self.assertRaises(exception.EvaluatorParseException, + evaluator.evaluate, + "1/*1") + + def test_nonnumber_comparison(self): + nonnumber = {'test': 'foo'} + request = {'test': 'bar'} + self.assertRaises( + exception.EvaluatorParseException, + evaluator.evaluate, + "nonnumber.test != request.test", + nonnumber=nonnumber, request=request) + + def test_div_zero(self): + self.assertRaises(exception.EvaluatorParseException, + evaluator.evaluate, + "7 / 0") diff --git a/manila/tests/scheduler/filters/test_driver.py b/manila/tests/scheduler/filters/test_driver.py new file mode 100644 index 0000000000..2aac031bdb --- /dev/null +++ b/manila/tests/scheduler/filters/test_driver.py @@ -0,0 +1,191 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_context import context + +from manila.scheduler.filters import driver +from manila import test +from manila.tests.scheduler import fakes + + +class HostFiltersTestCase(test.TestCase): + + def setUp(self): + super(HostFiltersTestCase, self).setUp() + self.context = context.RequestContext('fake', 'fake') + self.filter = driver.DriverFilter() + + def test_passing_function(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'filter_function': '1 == 1', + } + }) + + filter_properties = {'share_type': {}} + + self.assertTrue(self.filter.host_passes(host1, filter_properties)) + + def test_failing_function(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'filter_function': '1 == 2', + } + }) + + filter_properties = {'share_type': {}} + + self.assertFalse(self.filter.host_passes(host1, filter_properties)) + + def test_no_filter_function(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'filter_function': None, + } + }) + + filter_properties = {'share_type': {}} + + self.assertTrue(self.filter.host_passes(host1, filter_properties)) + + def test_not_implemented(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': {} + }) + + filter_properties = {'share_type': {}} + + self.assertTrue(self.filter.host_passes(host1, filter_properties)) + + def test_no_share_extra_specs(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'filter_function': '1 == 1', + } + }) + + filter_properties = {'share_type': {}} + + self.assertTrue(self.filter.host_passes(host1, filter_properties)) + + def test_extra_specs_wrong_backend(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'filter_function': '1 == 1', + } + }) + + filter_properties = { + 'share_type': { + 'extra_specs': { + 'share_backend_name': 'foo', + } + } + } + + self.assertFalse(self.filter.host_passes(host1, filter_properties)) + + def test_function_extra_spec_replacement(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'filter_function': 'extra.var == 1', + } + }) + + filter_properties = { + 'share_type': { + 'extra_specs': { + 'var': 1, + } + } + } + + self.assertTrue(self.filter.host_passes(host1, filter_properties)) + + def test_function_stats_replacement(self): + host1 = fakes.FakeHostState( + 'host1', { + 'total_capacity_gb': 100, + 'capabilities': { + 'filter_function': 'stats.total_capacity_gb < 200', + } + }) + + filter_properties = {'share_type': {}} + + self.assertTrue(self.filter.host_passes(host1, filter_properties)) + + def test_function_share_replacement(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'filter_function': 'share.size < 5', + } + }) + + filter_properties = { + 'request_spec': { + 'resource_properties': { + 'size': 1 + } + } + } + + self.assertTrue(self.filter.host_passes(host1, filter_properties)) + + def test_function_exception_caught(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'filter_function': '1 / 0 == 0', + } + }) + + filter_properties = {} + + self.assertFalse(self.filter.host_passes(host1, filter_properties)) + + def test_capabilities(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'foo': 10, + 'filter_function': 'capabilities.foo == 10', + }, + }) + + filter_properties = {} + + self.assertTrue(self.filter.host_passes(host1, filter_properties)) + + def test_wrong_capabilities(self): + host1 = fakes.FakeHostState( + 'host1', { + 'capabilities': { + 'bar': 10, + 'filter_function': 'capabilities.foo == 10', + }, + }) + + filter_properties = {} + + self.assertFalse(self.filter.host_passes(host1, filter_properties)) diff --git a/manila/tests/scheduler/weighers/test_goodness.py b/manila/tests/scheduler/weighers/test_goodness.py new file mode 100644 index 0000000000..87df8fd764 --- /dev/null +++ b/manila/tests/scheduler/weighers/test_goodness.py @@ -0,0 +1,180 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Goodness Weigher. +""" + +from manila.scheduler.weighers import goodness +from manila import test +from manila.tests.scheduler import fakes + + +class GoodnessWeigherTestCase(test.TestCase): + + def test_goodness_weigher_with_no_goodness_function(self): + weigher = goodness.GoodnessWeigher() + host_state = fakes.FakeHostState('host1', { + 'host': 'host.example.com', + 'capabilities': { + 'foo': '50' + } + }) + + weight_properties = {} + weight = weigher._weigh_object(host_state, weight_properties) + self.assertEqual(0, weight) + + def test_goodness_weigher_passing_host(self): + weigher = goodness.GoodnessWeigher() + host_state = fakes.FakeHostState('host1', { + 'host': 'host.example.com', + 'capabilities': { + 'goodness_function': '100' + } + }) + host_state_2 = fakes.FakeHostState('host2', { + 'host': 'host2.example.com', + 'capabilities': { + 'goodness_function': '0' + } + }) + host_state_3 = fakes.FakeHostState('host3', { + 'host': 'host3.example.com', + 'capabilities': { + 'goodness_function': '100 / 2' + } + }) + + weight_properties = {} + weight = weigher._weigh_object(host_state, weight_properties) + self.assertEqual(100, weight) + weight = weigher._weigh_object(host_state_2, weight_properties) + self.assertEqual(0, weight) + weight = weigher._weigh_object(host_state_3, weight_properties) + self.assertEqual(50, weight) + + def test_goodness_weigher_capabilities_substitution(self): + weigher = goodness.GoodnessWeigher() + host_state = fakes.FakeHostState('host1', { + 'host': 'host.example.com', + 'capabilities': { + 'foo': 50, + 'goodness_function': '10 + capabilities.foo' + } + }) + + weight_properties = {} + weight = weigher._weigh_object(host_state, weight_properties) + self.assertEqual(60, weight) + + def test_goodness_weigher_extra_specs_substitution(self): + weigher = goodness.GoodnessWeigher() + host_state = fakes.FakeHostState('host1', { + 'host': 'host.example.com', + 'capabilities': { + 'goodness_function': '10 + extra.foo' + } + }) + + weight_properties = { + 'share_type': { + 'extra_specs': { + 'foo': 50 + } + } + } + weight = weigher._weigh_object(host_state, weight_properties) + self.assertEqual(60, weight) + + def test_goodness_weigher_share_substitution(self): + weigher = goodness.GoodnessWeigher() + host_state = fakes.FakeHostState('host1', { + 'host': 'host.example.com', + 'capabilities': { + 'goodness_function': '10 + share.foo' + } + }) + + weight_properties = { + 'request_spec': { + 'resource_properties': { + 'foo': 50 + } + } + } + weight = weigher._weigh_object(host_state, weight_properties) + self.assertEqual(60, weight) + + def test_goodness_weigher_stats_substitution(self): + weigher = goodness.GoodnessWeigher() + host_state = fakes.FakeHostState('host1', { + 'host': 'host.example.com', + 'capabilities': { + 'goodness_function': 'stats.free_capacity_gb > 20' + }, + 'free_capacity_gb': 50 + }) + + weight_properties = {} + weight = weigher._weigh_object(host_state, weight_properties) + self.assertEqual(100, weight) + + def test_goodness_weigher_invalid_substitution(self): + weigher = goodness.GoodnessWeigher() + host_state = fakes.FakeHostState('host1', { + 'host': 'host.example.com', + 'capabilities': { + 'goodness_function': '10 + stats.my_val' + }, + 'foo': 50 + }) + + weight_properties = {} + weight = weigher._weigh_object(host_state, weight_properties) + self.assertEqual(0, weight) + + def test_goodness_weigher_host_rating_out_of_bounds(self): + weigher = goodness.GoodnessWeigher() + host_state = fakes.FakeHostState('host1', { + 'host': 'host.example.com', + 'capabilities': { + 'goodness_function': '-10' + } + }) + host_state_2 = fakes.FakeHostState('host2', { + 'host': 'host2.example.com', + 'capabilities': { + 'goodness_function': '200' + } + }) + + weight_properties = {} + weight = weigher._weigh_object(host_state, weight_properties) + self.assertEqual(0, weight) + weight = weigher._weigh_object(host_state_2, weight_properties) + self.assertEqual(0, weight) + + def test_goodness_weigher_invalid_goodness_function(self): + weigher = goodness.GoodnessWeigher() + host_state = fakes.FakeHostState('host1', { + 'host': 'host.example.com', + 'capabilities': { + 'goodness_function': '50 / 0' + } + }) + + weight_properties = {} + weight = weigher._weigh_object(host_state, weight_properties) + self.assertEqual(0, weight) diff --git a/manila/tests/share/drivers/emc/test_driver.py b/manila/tests/share/drivers/emc/test_driver.py index b2f68dcd60..5a7b92e2a7 100644 --- a/manila/tests/share/drivers/emc/test_driver.py +++ b/manila/tests/share/drivers/emc/test_driver.py @@ -125,6 +125,8 @@ class EMCShareFrameworkTestCase(test.TestCase): data['pools'] = None data['snapshot_support'] = True data['replication_domain'] = None + data['filter_function'] = None + data['goodness_function'] = None self.assertEqual(data, self.driver._stats) def _fake_safe_get(self, value): diff --git a/manila/tests/share/drivers/glusterfs/test_glusterfs_native.py b/manila/tests/share/drivers/glusterfs/test_glusterfs_native.py index 68fdd801e3..55de3b13ba 100644 --- a/manila/tests/share/drivers/glusterfs/test_glusterfs_native.py +++ b/manila/tests/share/drivers/glusterfs/test_glusterfs_native.py @@ -257,6 +257,8 @@ class GlusterfsNativeShareDriverTestCase(test.TestCase): 'pools': None, 'snapshot_support': True, 'replication_domain': None, + 'filter_function': None, + 'goodness_function': None, } self.assertEqual(test_data, self._driver._stats) diff --git a/manila/tests/share/drivers/hpe/test_hpe_3par_driver.py b/manila/tests/share/drivers/hpe/test_hpe_3par_driver.py index f75b518919..969000a3ba 100644 --- a/manila/tests/share/drivers/hpe/test_hpe_3par_driver.py +++ b/manila/tests/share/drivers/hpe/test_hpe_3par_driver.py @@ -58,6 +58,8 @@ class HPE3ParDriverTestCase(test.TestCase): self.conf.network_config_group = 'test_network_config_group' self.conf.admin_network_config_group = ( 'test_admin_network_config_group') + self.conf.filter_function = None + self.conf.goodness_function = None def safe_get(attr): try: @@ -560,6 +562,8 @@ class HPE3ParDriverTestCase(test.TestCase): 'vendor_name': 'HPE', 'pools': None, 'replication_domain': None, + 'filter_function': None, + 'goodness_function': None, } result = self.driver.get_share_stats(refresh=True) @@ -618,6 +622,8 @@ class HPE3ParDriverTestCase(test.TestCase): 'hp3par_flash_cache': False, 'snapshot_support': True, 'replication_domain': None, + 'filter_function': None, + 'goodness_function': None, } result = self.driver.get_share_stats(refresh=True) @@ -652,6 +658,8 @@ class HPE3ParDriverTestCase(test.TestCase): 'vendor_name': 'HPE', 'snapshot_support': True, 'replication_domain': None, + 'filter_function': None, + 'goodness_function': None, } result = self.driver.get_share_stats(refresh=True) diff --git a/manila/tests/share/drivers/huawei/test_huawei_nas.py b/manila/tests/share/drivers/huawei/test_huawei_nas.py index 7191ff6f02..b2ce7e32f1 100644 --- a/manila/tests/share/drivers/huawei/test_huawei_nas.py +++ b/manila/tests/share/drivers/huawei/test_huawei_nas.py @@ -746,6 +746,8 @@ class HuaweiShareDriverTestCase(test.TestCase): self.configuration.max_over_subscription_ratio = 1 self.configuration.driver_handles_share_servers = False self.configuration.replication_domain = None + self.configuration.filter_function = None + self.configuration.goodness_function = None self.tmp_dir = tempfile.mkdtemp() self.fake_conf_file = self.tmp_dir + '/manila_huawei_conf.xml' @@ -2189,6 +2191,8 @@ class HuaweiShareDriverTestCase(test.TestCase): expected['qos'] = True expected["snapshot_support"] = True expected['replication_domain'] = None + expected['filter_function'] = None + expected['goodness_function'] = None expected["pools"] = [] pool = dict( pool_name='OpenStack_Pool', diff --git a/manila/tests/share/drivers/zfsonlinux/test_driver.py b/manila/tests/share/drivers/zfsonlinux/test_driver.py index 8dc3d71477..0135b91a68 100644 --- a/manila/tests/share/drivers/zfsonlinux/test_driver.py +++ b/manila/tests/share/drivers/zfsonlinux/test_driver.py @@ -59,6 +59,8 @@ class FakeConfig(object): "reserved_share_percentage", 0) self.max_over_subscription_ratio = kwargs.get( "max_over_subscription_ratio", 15.0) + self.filter_function = kwargs.get("filter_function", None) + self.goodness_function = kwargs.get("goodness_function", None) def safe_get(self, key): return getattr(self, key) @@ -303,6 +305,8 @@ class ZFSonLinuxShareDriverTestCase(test.TestCase): 'storage_protocol': 'NFS', 'total_capacity_gb': 'unknown', 'vendor_name': 'Open Source', + 'filter_function': None, + 'goodness_function': None, } if replication_domain: expected['replication_type'] = 'readable' diff --git a/releasenotes/notes/driver-filter-91e2c60c9d1a48dd.yaml b/releasenotes/notes/driver-filter-91e2c60c9d1a48dd.yaml new file mode 100644 index 0000000000..0ad7af1735 --- /dev/null +++ b/releasenotes/notes/driver-filter-91e2c60c9d1a48dd.yaml @@ -0,0 +1,10 @@ +--- +features: + - Add DriverFilter and GoodnessWeigher to manila's scheduler. + These can use two new properties provided by backends, 'filter_function' + and 'goodness_function', which can be used to filter and weigh qualified + backends, respectively. +upgrade: + - To add DriverFilter and GoodnessWeigher to an active deployment, their + references must be added to the filters and weighers sections on + entry_points.txt. diff --git a/requirements.txt b/requirements.txt index 9ccc68509e..5b9ec9dba4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ oslo.concurrency>=3.8.0 # Apache-2.0 paramiko>=2.0 # LGPLv2.1+ Paste # MIT PasteDeploy>=1.5.0 # MIT +pyparsing>=2.0.1 # MIT python-neutronclient>=4.2.0 # Apache-2.0 keystoneauth1>=2.7.0 # Apache-2.0 keystonemiddleware!=4.1.0,!=4.5.0,>=4.0.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index ebcccb6886..95ab90381b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,12 +38,14 @@ manila.scheduler.filters = CapabilitiesFilter = manila.scheduler.filters.capabilities:CapabilitiesFilter CapacityFilter = manila.scheduler.filters.capacity:CapacityFilter ConsistencyGroupFilter = manila.scheduler.filters.consistency_group:ConsistencyGroupFilter + DriverFilter = manila.scheduler.filters.driver:DriverFilter IgnoreAttemptedHostsFilter = manila.scheduler.filters.ignore_attempted_hosts:IgnoreAttemptedHostsFilter JsonFilter = manila.scheduler.filters.json:JsonFilter RetryFilter = manila.scheduler.filters.retry:RetryFilter ShareReplicationFilter = manila.scheduler.filters.share_replication:ShareReplicationFilter manila.scheduler.weighers = CapacityWeigher = manila.scheduler.weighers.capacity:CapacityWeigher + GoodnessWeigher = manila.scheduler.weighers.goodness:GoodnessWeigher PoolWeigher = manila.scheduler.weighers.pool:PoolWeigher # These are for backwards compat with Havana notification_driver configuration values oslo_messaging.notify.drivers =