openstack
/
cinder
Code Issues Proposed changes

Sync scheduler module from oslo-incubator 

The scheduler and scheduler.weights modules have not been
updated since early in the Icehouse release cycle.  This
patch brings the version in Cinder up to date with what is in
oslo-incubator.

Current HEAD in OSLO:
---------------------
commit 20d7dc57819a70abdff967299542068946d75ac1
Date:   Wed Jul 29 03:49:46 2015 +0000
    Updated from global requirements

    Change-Id: Iba0e4f9545fc9ba82f080a0fec672761dcfeaeec

Patches included with sync by file:
---------------------
base_filter.py
 - 2fbf5065 - Remove oslo.log code and clean up versionutils API
 - 5d40e143 - Remove code that moved to oslo.i18n
 - 2af88ece - Use _LI instead of _ for info message translation
 - 4a47188e - Allow filters to only run once per request if their data is static
 - fcf517d7 - Update oslo log messages with translation domains
 - cae33101 - Stop looping all filters if no objects return
 base_handler.py
 - 6fa29aee - Trivial: Make vertical white space after license header consistent
 - 35660dac7 - Enable hacking H401 test
 - 5dcc43b1 - Break out common functionality for filters and weights
base_weight.py
 - 825cb870 - Upgrade to hacking 0.10
 - a2fa4878 - Fix common.scheduler.base_weight.BaseWeigher to be Python3 compat
 - e47bc70e - Normalize Scheduler Weights

 Note: Key changes were made that impact cinder:
   (1) The weigher base class changed the name of _weight_mulitipler() to
       weight_multiplier() which impacted cinder/scheduler/weights/capacity.py
       and cinder/scheduler/weights/volume_number.py.
   (2) Application of the weight multiplier was moved from the weigher to the
       handler and normalization of the weights prior to applying the weight
       multiplier was added.  This impacted cinder weigher test cases which
       were checking for non-normalized weights.
   (3) The normalization does not support the use of infinite weights which
       are used by the cinder capacity weigher.  When an infinite value is
       used, it is not known what the other weights will be, so this adds an
       override to CapacityWeigher for the weigh_objects() method that wraps
       the parent method to use the returned complete set of weights to
       replace any infinite weights with values that are much larger than
       the largest non-infinite weight.

Change-Id: Iad9bb431f0876e4957598c0ac0d3c8d7f67199fc
This commit is contained in:
James Carey 2015-07-29 22:55:03 +00:00
parent d5b9f39046
commit badedd29ea
8 changed files with 204 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
cinder/openstack/common/scheduler/base_filter.py
View File

@ -16,9 +16,13 @@
"""
Filter support
"""
import logging
from cinder.openstack.common._i18n import _LI
from cinder.openstack.common.scheduler import base_handler
LOG = logging.getLogger(__name__)
class BaseFilter(object):
"""Base class for all filter classes."""
@ -39,6 +43,17 @@ class BaseFilter(object):
if self._filter_one(obj, filter_properties):
yield obj
# Set to true in a subclass if a filter only needs to be run once
# for each request rather than for each instance
run_filter_once_per_request = False
def run_filter_for_index(self, index):
"""Return True if the filter needs to be run for the "index-th"
instance in a request. Only need to override this if a filter
needs anything other than "first only" or "all" behaviour.
"""
return not (self.run_filter_once_per_request and index > 0)
class BaseFilterHandler(base_handler.BaseHandler):
"""Base class to handle loading filter classes.
@ -47,7 +62,34 @@ class BaseFilterHandler(base_handler.BaseHandler):
"""
def get_filtered_objects(self, filter_classes, objs,
filter_properties):
filter_properties, index=0):
"""Get objects after filter
:param filter_classes: filters that will be used to filter the
objects
:param objs: objects that will be filtered
:param filter_properties: client filter properties
:param index: This value needs to be increased in the caller
function of get_filtered_objects when handling
each resource.
"""
list_objs = list(objs)
LOG.debug("Starting with %d host(s)", len(list_objs))
for filter_cls in filter_classes:
objs = filter_cls().filter_all(objs, filter_properties)
return list(objs)
cls_name = filter_cls.__name__
filter_class = filter_cls()
if filter_class.run_filter_for_index(index):
objs = filter_class.filter_all(list_objs, filter_properties)
if objs is None:
LOG.debug("Filter %(cls_name)s says to stop filtering",
{'cls_name': cls_name})
return
list_objs = list(objs)
msg = (_LI("Filter %(cls_name)s returned %(obj_len)d host(s)")
% {'cls_name': cls_name, 'obj_len': len(list_objs)})
if not list_objs:
LOG.info(msg)
break
LOG.debug(msg)
return list_objs

1
cinder/openstack/common/scheduler/base_handler.py
View File

@ -12,6 +12,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
A common base for handling extension classes.

99
cinder/openstack/common/scheduler/base_weight.py
View File

@ -17,9 +17,42 @@
Pluggable Weighing support
"""
import abc
import six
from cinder.openstack.common.scheduler import base_handler
def normalize(weight_list, minval=None, maxval=None):
"""Normalize the values in a list between 0 and 1.0.
The normalization is made regarding the lower and upper values present in
weight_list. If the minval and/or maxval parameters are set, these values
will be used instead of the minimum and maximum from the list.
If all the values are equal, they are normalized to 0.
"""
if not weight_list:
return ()
if maxval is None:
maxval = max(weight_list)
if minval is None:
minval = min(weight_list)
maxval = float(maxval)
minval = float(minval)
if minval == maxval:
return [0] * len(weight_list)
range_ = maxval - minval
return ((i - minval) / range_ for i in weight_list)
class WeighedObject(object):
"""Object with weight information."""
def __init__(self, obj, weight):
@ -30,28 +63,61 @@ class WeighedObject(object):
return "<WeighedObject '%s': %s>" % (self.obj, self.weight)
@six.add_metaclass(abc.ABCMeta)
class BaseWeigher(object):
"""Base class for pluggable weighers."""
def _weight_multiplier(self):
"""How weighted this weigher should be. Normally this would
be overridden in a subclass based on a config value.
"""Base class for pluggable weighers.
The attributes maxval and minval can be specified to set up the maximum
and minimum values for the weighed objects. These values will then be
taken into account in the normalization step, instead of taking the values
from the calculated weights.
"""
minval = None
maxval = None
def weight_multiplier(self):
"""How weighted this weigher should be.
Override this method in a subclass, so that the returned value is
read from a configuration option to permit operators specify a
multiplier for the weigher.
"""
return 1.0
@abc.abstractmethod
def _weigh_object(self, obj, weight_properties):
"""Override in a subclass to specify a weight for a specific
object.
"""
return 0.0
def weigh_objects(self, weighed_obj_list, weight_properties):
"""Weigh multiple objects. Override in a subclass if you need
need access to all objects in order to manipulate weights.
"""Weigh multiple objects.
Override in a subclass if you need access to all objects in order
to calculate weights. Do not modify the weight of an object here,
just return a list of weights.
"""
constant = self._weight_multiplier()
# Calculate the weights
weights = []
for obj in weighed_obj_list:
obj.weight += (constant *
self._weigh_object(obj.obj, weight_properties))
weight = self._weigh_object(obj.obj, weight_properties)
# Record the min and max values if they are None. If they anything
# but none we assume that the weigher has set them
if self.minval is None:
self.minval = weight
if self.maxval is None:
self.maxval = weight
if weight < self.minval:
self.minval = weight
elif weight > self.maxval:
self.maxval = weight
weights.append(weight)
return weights
class BaseWeightHandler(base_handler.BaseHandler):
@ -59,7 +125,7 @@ class BaseWeightHandler(base_handler.BaseHandler):
def get_weighed_objects(self, weigher_classes, obj_list,
weighing_properties):
"""Return a sorted (highest score first) list of WeighedObjects."""
"""Return a sorted (descending), normalized list of WeighedObjects."""
if not obj_list:
return []
@ -67,6 +133,15 @@ class BaseWeightHandler(base_handler.BaseHandler):
weighed_objs = [self.object_class(obj, 0.0) for obj in obj_list]
for weigher_cls in weigher_classes:
weigher = weigher_cls()
weigher.weigh_objects(weighed_objs, weighing_properties)
weights = weigher.weigh_objects(weighed_objs, weighing_properties)
# Normalize the weights
weights = normalize(weights,
minval=weigher.minval,
maxval=weigher.maxval)
for i, weight in enumerate(weights):
obj = weighed_objs[i]
obj.weight += weigher.weight_multiplier() * weight
return sorted(weighed_objs, key=lambda x: x.weight, reverse=True)

36
cinder/scheduler/weights/capacity.py
View File

@ -61,12 +61,44 @@ capacity_weight_opts = [
CONF = cfg.CONF
CONF.register_opts(capacity_weight_opts)
OFFSET_MIN = 10000
OFFSET_MULT = 100
class CapacityWeigher(weights.BaseHostWeigher):
def _weight_multiplier(self):
def weight_multiplier(self):
"""Override the weight multiplier."""
return CONF.capacity_weight_multiplier
def weigh_objects(self, weighed_obj_list, weight_properties):
"""Override the weigh objects.
This override calls the parent to do the weigh objects and then
replaces any infinite weights with a value that is a multiple of the
delta between the min and max values.
NOTE(jecarey): the infinite weight value is only used when the
smallest value is being favored (negative multiplier). When the
largest weight value is being used a weight of -1 is used instead.
See _weigh_object method.
"""
tmp_weights = super(weights.BaseHostWeigher, self).weigh_objects(
weighed_obj_list, weight_properties)
if math.isinf(self.maxval):
# NOTE(jecarey): if all weights were infinite then parent
# method returns 0 for all of the weights. Thus self.minval
# cannot be infinite at this point
copy_weights = [w for w in tmp_weights if not math.isinf(w)]
self.maxval = max(copy_weights)
offset = (self.maxval - self.minval) * OFFSET_MULT
self.maxval += OFFSET_MIN if offset == 0.0 else offset
tmp_weights = [self.maxval if math.isinf(w) else w
for w in tmp_weights]
return tmp_weights
def _weigh_object(self, host_state, weight_properties):
"""Higher weights win. We want spreading to be the default."""
reserved = float(host_state.reserved_percentage) / 100
@ -96,7 +128,7 @@ class CapacityWeigher(weights.BaseHostWeigher):
class AllocatedCapacityWeigher(weights.BaseHostWeigher):
def _weight_multiplier(self):
def weight_multiplier(self):
"""Override the weight multiplier."""
return CONF.allocated_capacity_weight_multiplier

2
cinder/scheduler/weights/volume_number.py
View File

@ -44,7 +44,7 @@ CONF.register_opts(volume_number_weight_opts)
class VolumeNumberWeigher(weights.BaseHostWeigher):
def _weight_multiplier(self):
def weight_multiplier(self):
"""Override the weight multiplier."""
return CONF.volume_number_multiplier

21
cinder/tests/unit/scheduler/test_allocated_capacity_weigher.py
View File

@ -57,14 +57,15 @@ class AllocatedCapacityWeigherTestCase(test.TestCase):
def test_default_of_spreading_first(self):
hostinfo_list = self._get_all_hosts()
# host1: allocated_capacity_gb=0, weight=0
# host1: allocated_capacity_gb=0, weight=0 Norm=0.0
# host2: allocated_capacity_gb=1748, weight=-1748
# host3: allocated_capacity_gb=256, weight=-256
# host4: allocated_capacity_gb=1848, weight=-1848
# host4: allocated_capacity_gb=1848, weight=-1848 Norm=-1.0
# host5: allocated_capacity_gb=1548, weight=-1540
# so, host1 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(0, weighed_host.weight)
self.assertEqual(0.0, weighed_host.weight)
self.assertEqual(
'host1', utils.extract_host(weighed_host.obj.host))
@ -72,14 +73,15 @@ class AllocatedCapacityWeigherTestCase(test.TestCase):
self.flags(allocated_capacity_weight_multiplier=1.0)
hostinfo_list = self._get_all_hosts()
# host1: allocated_capacity_gb=0, weight=0
# host1: allocated_capacity_gb=0, weight=0 Norm=0.0
# host2: allocated_capacity_gb=1748, weight=1748
# host3: allocated_capacity_gb=256, weight=256
# host4: allocated_capacity_gb=1848, weight=1848
# host4: allocated_capacity_gb=1848, weight=1848 Norm=1.0
# host5: allocated_capacity_gb=1548, weight=1540
# so, host4 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(1848.0, weighed_host.weight)
self.assertEqual(1.0, weighed_host.weight)
self.assertEqual(
'host4', utils.extract_host(weighed_host.obj.host))
@ -87,13 +89,14 @@ class AllocatedCapacityWeigherTestCase(test.TestCase):
self.flags(allocated_capacity_weight_multiplier=-2.0)
hostinfo_list = self._get_all_hosts()
# host1: allocated_capacity_gb=0, weight=0
# host1: allocated_capacity_gb=0, weight=0 Norm=0.0
# host2: allocated_capacity_gb=1748, weight=-3496
# host3: allocated_capacity_gb=256, weight=-512
# host4: allocated_capacity_gb=1848, weight=-3696
# host4: allocated_capacity_gb=1848, weight=-3696 Norm=-2.0
# host5: allocated_capacity_gb=1548, weight=-3080
# so, host1 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 0)
self.assertEqual(0.0, weighed_host.weight)
self.assertEqual(
'host1', utils.extract_host(weighed_host.obj.host))

21
cinder/tests/unit/scheduler/test_capacity_weigher.py
View File

@ -66,19 +66,24 @@ class CapacityWeigherTestCase(test.TestCase):
# host1: thin_provisioning_support = False
# free_capacity_gb=1024,
# free=1024-math.floor(1024*0.1)=922
# Norm=0.837837837838
# host2: thin_provisioning_support = True
# free_capacity_gb=300,
# free=2048*1.5-1748-math.floor(2048*0.1)=1120
# Norm=1.0
# host3: thin_provisioning_support = False
# free_capacity_gb=512, free=256-512*0=256
# Norm=0.292383292383
# host4: thin_provisioning_support = True
# free_capacity_gb=200,
# free=2048*1.0-2047-math.floor(2048*0.05)=-101
# Norm=0.0
# host5: free_capacity_gb=unknown free=-1
# Norm=0.0819000819001
# so, host2 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 1120.0)
self.assertEqual(weighed_host.weight, 1.0)
self.assertEqual(
utils.extract_host(weighed_host.obj.host), 'host2')
@ -89,19 +94,24 @@ class CapacityWeigherTestCase(test.TestCase):
# host1: thin_provisioning_support = False
# free_capacity_gb=1024,
# free=-(1024-math.floor(1024*0.1))=-922
# Norm=-0.00829542413701
# host2: thin_provisioning_support = True
# free_capacity_gb=300,
# free=-(2048*1.5-1748-math.floor(2048*0.1))=-1120
# Norm=-0.00990099009901
# host3: thin_provisioning_support = False
# free_capacity_gb=512, free=-(256-512*0)=-256
# Norm=--0.002894884083
# host4: thin_provisioning_support = True
# free_capacity_gb=200,
# free=-(2048*1.0-2047-math.floor(2048*0.05))=101
# Norm=0.0
# host5: free_capacity_gb=unknown free=-float('inf')
# Norm=-1.0
# so, host4 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 101.0)
self.assertEqual(weighed_host.weight, 0.0)
self.assertEqual(
utils.extract_host(weighed_host.obj.host), 'host4')
@ -112,18 +122,23 @@ class CapacityWeigherTestCase(test.TestCase):
# host1: thin_provisioning_support = False
# free_capacity_gb=1024,
# free=(1024-math.floor(1024*0.1))*2=1844
# Norm=1.67567567568
# host2: thin_provisioning_support = True
# free_capacity_gb=300,
# free=(2048*1.5-1748-math.floor(2048*0.1))*2=2240
# Norm=2.0
# host3: thin_provisioning_support = False
# free_capacity_gb=512, free=(256-512*0)*2=512
# Norm=0.584766584767
# host4: thin_provisioning_support = True
# free_capacity_gb=200,
# free=(2048*1.0-2047-math.floor(2048*0.05))*2=-202
# Norm=0.0
# host5: free_capacity_gb=unknown free=-2
# Norm=0.1638001638
# so, host2 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 1120.0 * 2)
self.assertEqual(weighed_host.weight, 1.0 * 2)
self.assertEqual(
utils.extract_host(weighed_host.obj.host), 'host2')

12
cinder/tests/unit/scheduler/test_volume_number_weigher.py
View File

@ -76,16 +76,16 @@ class VolumeNumberWeigherTestCase(test.TestCase):
self.flags(volume_number_multiplier=-1.0)
hostinfo_list = self._get_all_hosts()
# host1: 1 volume
# host1: 1 volume Norm=0.0
# host2: 2 volumes
# host3: 3 volumes
# host4: 4 volumes
# host5: 5 volumes
# host5: 5 volumes Norm=-1.0
# so, host1 should win:
with mock.patch.object(api, 'volume_data_get_for_host',
fake_volume_data_get_for_host):
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, -1.0)
self.assertEqual(weighed_host.weight, 0.0)
self.assertEqual(utils.extract_host(weighed_host.obj.host),
'host1')
@ -93,15 +93,15 @@ class VolumeNumberWeigherTestCase(test.TestCase):
self.flags(volume_number_multiplier=1.0)
hostinfo_list = self._get_all_hosts()
# host1: 1 volume
# host1: 1 volume Norm=0
# host2: 2 volumes
# host3: 3 volumes
# host4: 4 volumes
# host5: 5 volumes
# host5: 5 volumes Norm=1
# so, host5 should win:
with mock.patch.object(api, 'volume_data_get_for_host',
fake_volume_data_get_for_host):
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 5.0)
self.assertEqual(weighed_host.weight, 1.0)
self.assertEqual(utils.extract_host(weighed_host.obj.host),
'host5')