x/gantt
Code Issues Proposed changes

Refactor scheduling weights.

Browse Source 
This makes scheduling weights more plugin friendly and creates shared
code that can be used by the host scheduler as well as the future cells
scheduler.  Weighing classes can now be specified much like you can
specify scheduling host filters.

The new weights code reverses the old behavior where lower weights win.
Higher weights are now the winners.

The least_cost module and configs have been deprecated, but are still
supported for backwards compatibility.  The code has moved to
nova.scheduler.weights.least_cost and been modified to work with the new
loadable-class code.  If any of the least_cost related config options are
specified, this least_cost weigher will be used.

For those not overriding the default least_cost config values, the new
RamWeigher class will be used.  The default behavior of the RamWeigher
class is the same default behavior as the old least_cost module.

The new weights code introduces a new config option
'scheduler_weight_classes' which is used to specify which weigher classes
to use.  The default is 'all classes', but modified if least_cost
deprecated config options are used, as mentioned above.

The RamWeigher class introduces a new config option
'ram_weight_multiplier'.  The default of 1.0 causes weights equal to the
free memory in MB to be returned, thus hosts with more free memory are
preferred (causes equal spreading).  Changing this value to a negative
number such as -1.0 will cause reverse behavior (fill first).

DocImpact

Change-Id: I1e5e5039c299db02f7287f2d33299ebf0b9732ce
This commit is contained in:
Chris Behrens 2012-11-09 02:48:09 +00:00
parent 770acb12ab
commit c728941c3d
10 changed files with 543 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
nova/scheduler/filter_scheduler.py
View File

@ -19,16 +19,11 @@ You can customize this scheduler by specifying your own Host Filters and
Weighing Functions.
"""
import operator
from nova import config
from nova import exception
from nova import flags
from nova.openstack.common import importutils
from nova.openstack.common import log as logging
from nova.openstack.common.notifier import api as notifier
from nova.scheduler import driver
from nova.scheduler import least_cost
from nova.scheduler import scheduler_options
CONF = config.CONF
@ -61,7 +56,7 @@ class FilterScheduler(driver.Scheduler):
notifier.notify(context, notifier.publisher_id("scheduler"),
'scheduler.run_instance.start', notifier.INFO, payload)
weighted_hosts = self._schedule(context, request_spec,
weighed_hosts = self._schedule(context, request_spec,
filter_properties, instance_uuids)
# NOTE(comstud): Make sure we do not pass this through. It
@ -73,11 +68,11 @@ class FilterScheduler(driver.Scheduler):
try:
try:
weighted_host = weighted_hosts.pop(0)
weighed_host = weighed_hosts.pop(0)
except IndexError:
raise exception.NoValidHost(reason="")
self._provision_resource(context, weighted_host,
self._provision_resource(context, weighed_host,
request_spec,
filter_properties,
requested_networks,
@ -107,29 +102,29 @@ class FilterScheduler(driver.Scheduler):
the prep_resize operation to it.
"""
hosts = self._schedule(context, request_spec, filter_properties,
[instance['uuid']])
if not hosts:
weighed_hosts = self._schedule(context, request_spec,
filter_properties, [instance['uuid']])
if not weighed_hosts:
raise exception.NoValidHost(reason="")
host = hosts.pop(0)
weighed_host = weighed_hosts.pop(0)
self._post_select_populate_filter_properties(filter_properties,
host.host_state)
weighed_host.obj)
# context is not serializable
filter_properties.pop('context', None)
# Forward off to the host
self.compute_rpcapi.prep_resize(context, image, instance,
instance_type, host.host_state.host, reservations,
instance_type, weighed_host.obj.host, reservations,
request_spec=request_spec, filter_properties=filter_properties)
def _provision_resource(self, context, weighted_host, request_spec,
def _provision_resource(self, context, weighed_host, request_spec,
filter_properties, requested_networks, injected_files,
admin_password, is_first_time, instance_uuid=None):
"""Create the requested resource in this Zone."""
payload = dict(request_spec=request_spec,
weighted_host=weighted_host.to_dict(),
weighted_host=weighed_host.to_dict(),
instance_id=instance_uuid)
notifier.notify(context, notifier.publisher_id("scheduler"),
'scheduler.run_instance.scheduled', notifier.INFO,
@ -137,15 +132,15 @@ class FilterScheduler(driver.Scheduler):
# TODO(NTTdocomo): Combine the next two updates into one
driver.db_instance_node_set(context,
instance_uuid, weighted_host.host_state.nodename)
instance_uuid, weighed_host.obj.nodename)
updated_instance = driver.instance_update_db(context,
instance_uuid)
self._post_select_populate_filter_properties(filter_properties,
weighted_host.host_state)
weighed_host.obj)
self.compute_rpcapi.run_instance(context, instance=updated_instance,
host=weighted_host.host_state.host,
host=weighed_host.obj.host,
request_spec=request_spec, filter_properties=filter_properties,
requested_networks=requested_networks,
injected_files=injected_files,
@ -232,7 +227,6 @@ class FilterScheduler(driver.Scheduler):
instance_properties = request_spec['instance_properties']
instance_type = request_spec.get("instance_type", None)
cost_functions = self.get_cost_functions()
config_options = self._get_configuration_options()
# check retry policy. Rather ugly use of instance_uuids[0]...
@ -276,60 +270,12 @@ class FilterScheduler(driver.Scheduler):
LOG.debug(_("Filtered %(hosts)s") % locals())
# weighted_host = WeightedHost() ... the best
# host for the job.
# TODO(comstud): filter_properties will also be used for
# weighing and I plan fold weighing into the host manager
# in a future patch. I'll address the naming of this
# variable at that time.
weighted_host = least_cost.weighted_sum(cost_functions,
hosts, filter_properties)
LOG.debug(_("Weighted %(weighted_host)s") % locals())
selected_hosts.append(weighted_host)
weighed_hosts = self.host_manager.get_weighed_hosts(hosts,
filter_properties)
best_host = weighed_hosts[0]
LOG.debug(_("Choosing host %(best_host)s") % locals())
selected_hosts.append(best_host)
# Now consume the resources so the filter/weights
# will change for the next instance.
weighted_host.host_state.consume_from_instance(
instance_properties)
selected_hosts.sort(key=operator.attrgetter('weight'))
best_host.obj.consume_from_instance(instance_properties)
return selected_hosts
def get_cost_functions(self):
"""Returns a list of tuples containing weights and cost functions to
use for weighing hosts
"""
if self.cost_function_cache is not None:
return self.cost_function_cache
cost_fns = []
for cost_fn_str in CONF.least_cost_functions:
if '.' in cost_fn_str:
short_name = cost_fn_str.split('.')[-1]
else:
short_name = cost_fn_str
cost_fn_str = "%s.%s.%s" % (
__name__, self.__class__.__name__, short_name)
if not (short_name.startswith('compute_') or
short_name.startswith('noop')):
continue
try:
# NOTE: import_class is somewhat misnamed since
# the weighing function can be any non-class callable
# (i.e., no 'self')
cost_fn = importutils.import_class(cost_fn_str)
except ImportError:
raise exception.SchedulerCostFunctionNotFound(
cost_fn_str=cost_fn_str)
try:
flag_name = "%s_weight" % cost_fn.__name__
weight = getattr(CONF, flag_name)
except AttributeError:
raise exception.SchedulerWeightFlagNotFound(
flag_name=flag_name)
cost_fns.append((weight, cost_fn))
self.cost_function_cache = cost_fns
return cost_fns

12
nova/scheduler/host_manager.py
View File

@ -28,6 +28,7 @@ from nova.openstack.common import cfg
from nova.openstack.common import log as logging
from nova.openstack.common import timeutils
from nova.scheduler import filters
from nova.scheduler import weights
host_manager_opts = [
cfg.MultiStrOpt('scheduler_available_filters',
@ -47,6 +48,9 @@ host_manager_opts = [
],
help='Which filter class names to use for filtering hosts '
'when not specified in the request.'),
cfg.ListOpt('scheduler_weight_classes',
default=['nova.scheduler.weights.all_weighers'],
help='Which weight class names to use for weighing hosts'),
]
CONF = config.CONF
@ -258,6 +262,9 @@ class HostManager(object):
self.filter_handler = filters.HostFilterHandler()
self.filter_classes = self.filter_handler.get_matching_classes(
CONF.scheduler_available_filters)
self.weight_handler = weights.HostWeightHandler()
self.weight_classes = self.weight_handler.get_matching_classes(
CONF.scheduler_weight_classes)
def _choose_host_filters(self, filter_cls_names):
"""Since the caller may specify which filters to use we need
@ -316,6 +323,11 @@ class HostManager(object):
return self.filter_handler.get_filtered_objects(filter_classes,
hosts, filter_properties)
def get_weighed_hosts(self, hosts, weight_properties):
"""Weigh the hosts"""
return self.weight_handler.get_weighed_objects(self.weight_classes,
hosts, weight_properties)
def update_service_capabilities(self, service_name, host, capabilities):
"""Update the per-service capabilities based on this notification."""

118
nova/scheduler/least_cost.py
View File

@ -1,118 +0,0 @@
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Least Cost is an algorithm for choosing which host machines to
provision a set of resources to. The input is a WeightedHost object which
is decided upon by a set of objective-functions, called the 'cost-functions'.
The WeightedHost contains a combined weight for each cost-function.
The cost-function and weights are tabulated, and the host with the least cost
is then selected for provisioning.
"""
from nova import config
from nova import flags
from nova.openstack.common import cfg
from nova.openstack.common import log as logging
LOG = logging.getLogger(__name__)
least_cost_opts = [
cfg.ListOpt('least_cost_functions',
default=[
'nova.scheduler.least_cost.compute_fill_first_cost_fn'
],
help='Which cost functions the LeastCostScheduler should use'),
cfg.FloatOpt('noop_cost_fn_weight',
default=1.0,
help='How much weight to give the noop cost function'),
cfg.FloatOpt('compute_fill_first_cost_fn_weight',
default=-1.0,
help='How much weight to give the fill-first cost function. '
'A negative value will reverse behavior: '
'e.g. spread-first'),
]
CONF = config.CONF
CONF.register_opts(least_cost_opts)
# TODO(sirp): Once we have enough of these rules, we can break them out into a
# cost_functions.py file (perhaps in a least_cost_scheduler directory)
class WeightedHost(object):
"""Reduced set of information about a host that has been weighed.
This is an attempt to remove some of the ad-hoc dict structures
previously used."""
def __init__(self, weight, host_state=None):
self.weight = weight
self.host_state = host_state
def to_dict(self):
x = dict(weight=self.weight)
if self.host_state:
x['host'] = self.host_state.host
return x
def __repr__(self):
if self.host_state:
return "WeightedHost host: %s" % self.host_state.host
return "WeightedHost with no host_state"
def noop_cost_fn(host_state, weighing_properties):
"""Return a pre-weight cost of 1 for each host"""
return 1
def compute_fill_first_cost_fn(host_state, weighing_properties):
"""More free ram = higher weight. So servers with less free
ram will be preferred.
Note: the weight for this function in default configuration
is -1.0. With a -1.0 this function runs in reverse, so systems
with the most free memory will be preferred.
"""
return host_state.free_ram_mb
def weighted_sum(weighted_fns, host_states, weighing_properties):
"""Use the weighted-sum method to compute a score for an array of objects.
Normalize the results of the objective-functions so that the weights are
meaningful regardless of objective-function's range.
:param host_list: ``[(host, HostInfo()), ...]``
:param weighted_fns: list of weights and functions like::
[(weight, objective-functions), ...]
:param weighing_properties: an arbitrary dict of values that can
influence weights.
:returns: a single WeightedHost object which represents the best
candidate.
"""
min_score, best_host = None, None
for host_state in host_states:
score = sum(weight * fn(host_state, weighing_properties)
for weight, fn in weighted_fns)
if min_score is None or score < min_score:
min_score, best_host = score, host_state
return WeightedHost(min_score, host_state=best_host)

61
nova/scheduler/weights/__init__.py Normal file
View File

@ -0,0 +1,61 @@
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Scheduler host weights
"""
from nova import config
from nova.openstack.common import log as logging
from nova.scheduler.weights import least_cost
from nova import weights
LOG = logging.getLogger(__name__)
CONF = config.CONF
class WeighedHost(weights.WeighedObject):
def to_dict(self):
x = dict(weight=self.weight)
x['host'] = self.obj.host
return x
def __repr__(self):
return "WeighedHost [host: %s, weight: %s]" % (
self.obj.host, self.weight)
class BaseHostWeigher(weights.BaseWeigher):
"""Base class for host weights."""
pass
class HostWeightHandler(weights.BaseWeightHandler):
object_class = WeighedHost
def __init__(self):
super(HostWeightHandler, self).__init__(BaseHostWeigher)
def all_weighers():
"""Return a list of weight plugin classes found in this directory."""
if (CONF.least_cost_functions is not None or
CONF.compute_fill_first_cost_fn_weight is not None):
LOG.deprecated(_('least_cost has been deprecated in favor of '
'the RAM Weigher.'))
return least_cost.get_least_cost_weighers()
return HostWeightHandler().get_all_classes()

126
nova/scheduler/weights/least_cost.py Normal file
View File

@ -0,0 +1,126 @@
# Copyright (c) 2011-2012 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Least Cost is an algorithm for choosing which host machines to
provision a set of resources to. The input is a WeightedHost object which
is decided upon by a set of objective-functions, called the 'cost-functions'.
The WeightedHost contains a combined weight for each cost-function.
The cost-function and weights are tabulated, and the host with the least cost
is then selected for provisioning.
NOTE(comstud): This is deprecated. One should use the RAMWeigher and/or
create other weight modules.
"""
from nova import config
from nova import exception
from nova.openstack.common import cfg
from nova.openstack.common import importutils
from nova.openstack.common import log as logging
LOG = logging.getLogger(__name__)
least_cost_opts = [
cfg.ListOpt('least_cost_functions',
default=None,
help='Which cost functions the LeastCostScheduler should use'),
cfg.FloatOpt('noop_cost_fn_weight',
default=1.0,
help='How much weight to give the noop cost function'),
cfg.FloatOpt('compute_fill_first_cost_fn_weight',
default=None,
help='How much weight to give the fill-first cost function. '
'A negative value will reverse behavior: '
'e.g. spread-first'),
]
CONF = config.CONF
CONF.register_opts(least_cost_opts)
def noop_cost_fn(host_state, weight_properties):
"""Return a pre-weight cost of 1 for each host"""
return 1
def compute_fill_first_cost_fn(host_state, weight_properties):
"""Higher weights win, so we should return a lower weight
when there's more free ram available.
Note: the weight modifier for this function in default configuration
is -1.0. With -1.0 this function runs in reverse, so systems
with the most free memory will be preferred.
"""
return -host_state.free_ram_mb
def _get_cost_functions():
"""Returns a list of tuples containing weights and cost functions to
use for weighing hosts
"""
cost_fns_conf = CONF.least_cost_functions
if cost_fns_conf is None:
# The old default. This will get fixed up below.
fn_str = 'nova.scheduler.least_cost.compute_fill_first_cost_fn'
cost_fns_conf = [fn_str]
cost_fns = []
for cost_fn_str in cost_fns_conf:
short_name = cost_fn_str.split('.')[-1]
if not (short_name.startswith('compute_') or
short_name.startswith('noop')):
continue
# Fix up any old paths to the new paths
if cost_fn_str.startswith('nova.scheduler.least_cost.'):
cost_fn_str = ('nova.scheduler.weights.least_cost' +
cost_fn_str[25:])
try:
# NOTE: import_class is somewhat misnamed since
# the weighing function can be any non-class callable
# (i.e., no 'self')
cost_fn = importutils.import_class(cost_fn_str)
except ImportError:
raise exception.SchedulerCostFunctionNotFound(
cost_fn_str=cost_fn_str)
try:
flag_name = "%s_weight" % cost_fn.__name__
weight = getattr(CONF, flag_name)
except AttributeError:
raise exception.SchedulerWeightFlagNotFound(
flag_name=flag_name)
# Set the original default.
if (flag_name == 'compute_fill_first_cost_fn_weight' and
weight is None):
weight = -1.0
cost_fns.append((weight, cost_fn))
return cost_fns
def get_least_cost_weighers():
cost_functions = _get_cost_functions()
# Unfortunately we need to import this late so we don't have an
# import loop.
from nova.scheduler import weights
class _LeastCostWeigher(weights.BaseHostWeigher):
def weigh_objects(self, weighted_hosts, weight_properties):
for host in weighted_hosts:
host.weight = sum(weight * fn(host.obj, weight_properties)
for weight, fn in cost_functions)
return [_LeastCostWeigher]

46
nova/scheduler/weights/ram.py Normal file
View File

@ -0,0 +1,46 @@
# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
RAM Weigher. Weigh hosts by their RAM usage.
The default is to spread instances across all hosts evenly. If you prefer
stacking, you can set the 'ram_weight_multiplier' option to a negative
number and the weighing has the opposite effect of the default.
"""
from nova import config
from nova.openstack.common import cfg
from nova.scheduler import weights
ram_weight_opts = [
cfg.FloatOpt('ram_weight_multiplier',
default=1.0,
help='Multiplier used for weighing ram. Negative '
'numbers mean to stack vs spread.'),
]
CONF = config.CONF
CONF.register_opts(ram_weight_opts)
class RAMWeigher(weights.BaseHostWeigher):
def _weight_multiplier(self):
"""Override the weight multiplier."""
return CONF.ram_weight_multiplier
def _weigh_object(self, host_state, weight_properties):
"""Higher weights win. We want spreading to be the default."""
return host_state.free_ram_mb

41
nova/tests/scheduler/test_filter_scheduler.py
View File

@ -27,7 +27,7 @@ from nova import exception
from nova.scheduler import driver
from nova.scheduler import filter_scheduler
from nova.scheduler import host_manager
from nova.scheduler import least_cost
from nova.scheduler import weights
from nova.tests.scheduler import fakes
from nova.tests.scheduler import test_scheduler
@ -145,11 +145,10 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
self.next_weight = 1.0
def _fake_weighted_sum(functions, hosts, options):
def _fake_weigh_objects(_self, functions, hosts, options):
self.next_weight += 2.0
host_state = hosts[0]
return least_cost.WeightedHost(self.next_weight,
host_state=host_state)
return [weights.WeighedHost(host_state, self.next_weight)]
sched = fakes.FakeFilterScheduler()
fake_context = context.RequestContext('user', 'project',
@ -157,7 +156,8 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
self.stubs.Set(sched.host_manager, 'get_filtered_hosts',
fake_get_filtered_hosts)
self.stubs.Set(least_cost, 'weighted_sum', _fake_weighted_sum)
self.stubs.Set(weights.HostWeightHandler,
'get_weighed_objects', _fake_weigh_objects)
fakes.mox_host_manager_db_calls(self.mox, fake_context)
request_spec = {'num_instances': 10,
@ -171,10 +171,10 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
'vcpus': 1,
'os_type': 'Linux'}}
self.mox.ReplayAll()
weighted_hosts = sched._schedule(fake_context, request_spec, {})
self.assertEquals(len(weighted_hosts), 10)
for weighted_host in weighted_hosts:
self.assertTrue(weighted_host.host_state is not None)
weighed_hosts = sched._schedule(fake_context, request_spec, {})
self.assertEquals(len(weighed_hosts), 10)
for weighed_host in weighed_hosts:
self.assertTrue(weighed_host.obj is not None)
def test_schedule_prep_resize_doesnt_update_host(self):
fake_context = context.RequestContext('user', 'project',
@ -184,7 +184,7 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
def _return_hosts(*args, **kwargs):
host_state = host_manager.HostState('host2', 'node2')
return [least_cost.WeightedHost(1.0, host_state=host_state)]
return [weights.WeighedHost(host_state, 1.0)]
self.stubs.Set(sched, '_schedule', _return_hosts)
@ -203,19 +203,6 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
instance, {}, None)
self.assertEqual(info['called'], 0)
def test_get_cost_functions(self):
fixture = fakes.FakeFilterScheduler()
fns = fixture.get_cost_functions()
self.assertEquals(len(fns), 1)
weight, fn = fns[0]
self.assertEquals(weight, -1.0)
hostinfo = host_manager.HostState('host', 'node')
hostinfo.update_from_compute_node(dict(memory_mb=1000,
local_gb=0, vcpus=1, disk_available_least=1000,
free_disk_mb=1000, free_ram_mb=872, vcpus_used=0,
local_gb_used=0, updated_at=None))
self.assertEquals(872, fn(hostinfo, {}))
def test_max_attempts(self):
self.flags(scheduler_max_attempts=4)
@ -332,14 +319,14 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
reservations = None
host = fakes.FakeHostState('host', 'node', {})
weighted_host = least_cost.WeightedHost(1, host)
hosts = [weighted_host]
weighed_host = weights.WeighedHost(host, 1)
weighed_hosts = [weighed_host]
self.mox.StubOutWithMock(sched, '_schedule')
self.mox.StubOutWithMock(sched.compute_rpcapi, 'prep_resize')
sched._schedule(self.context, request_spec,
filter_properties, [instance['uuid']]).AndReturn(hosts)
sched._schedule(self.context, request_spec, filter_properties,
[instance['uuid']]).AndReturn(weighed_hosts)
sched.compute_rpcapi.prep_resize(self.context, image, instance,
instance_type, 'host', reservations, request_spec=request_spec,
filter_properties=filter_properties)

112
nova/tests/scheduler/test_least_cost.py
View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack LLC.
# Copyright 2011-2012 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -15,27 +15,51 @@
"""
Tests For Least Cost functions.
"""
from nova import config
from nova import context
from nova.scheduler import host_manager
from nova.scheduler import least_cost
from nova.openstack.common import cfg
from nova.scheduler import weights
from nova.scheduler.weights import least_cost
from nova import test
from nova.tests import matchers
from nova.tests.scheduler import fakes
def offset(hostinfo, options):
test_least_cost_opts = [
cfg.FloatOpt('compute_fake_weigher1_weight',
default=2.0,
help='How much weight to give the fake_weigher1 function'),
cfg.FloatOpt('compute_fake_weigher2_weight',
default=1.0,
help='How much weight to give the fake_weigher2 function'),
]
CONF = config.CONF
CONF.import_opt('least_cost_functions', 'nova.scheduler.weights.least_cost')
CONF.import_opt('compute_fill_first_cost_fn_weight',
'nova.scheduler.weights.least_cost')
CONF.register_opts(test_least_cost_opts)
def compute_fake_weigher1(hostinfo, options):
return hostinfo.free_ram_mb + 10000
def scale(hostinfo, options):
def compute_fake_weigher2(hostinfo, options):
return hostinfo.free_ram_mb * 2
class LeastCostTestCase(test.TestCase):
def setUp(self):
super(LeastCostTestCase, self).setUp()
self.flags(reserved_host_disk_mb=0, reserved_host_memory_mb=0)
self.host_manager = fakes.FakeHostManager()
self.weight_handler = weights.HostWeightHandler()
def _get_weighed_host(self, hosts, weight_properties=None):
weigher_classes = least_cost.get_least_cost_weighers()
if weight_properties is None:
weight_properties = {}
return self.weight_handler.get_weighed_objects(weigher_classes,
hosts, weight_properties)[0]
def _get_all_hosts(self):
ctxt = context.get_admin_context()
@ -46,8 +70,39 @@ class LeastCostTestCase(test.TestCase):
self.mox.ResetAll()
return host_states
def test_weighted_sum_happy_day(self):
fn_tuples = [(1.0, offset), (1.0, scale)]
def test_default_of_spread_first(self):
# Default modifier is -1.0, so it turns out that hosts with
# the most free memory win
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=512
# host2: free_ram_mb=1024
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# so, host1 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 8192)
self.assertEqual(weighed_host.obj.host, 'host4')
def test_filling_first(self):
self.flags(compute_fill_first_cost_fn_weight=1.0)
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=-512
# host2: free_ram_mb=-1024
# host3: free_ram_mb=-3072
# host4: free_ram_mb=-8192
# so, host1 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, -512)
self.assertEqual(weighed_host.obj.host, 'host1')
def test_weighted_sum_provided_method(self):
fns = ['nova.tests.scheduler.test_least_cost.compute_fake_weigher1',
'nova.tests.scheduler.test_least_cost.compute_fake_weigher2']
self.flags(least_cost_functions=fns)
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=512
@ -59,18 +114,17 @@ class LeastCostTestCase(test.TestCase):
# [10512, 11024, 13072, 18192]
# [1024, 2048, 6144, 16384]
# adjusted [ 1.0 * x + 1.0 * y] =
# [11536, 13072, 19216, 34576]
# adjusted [ 2.0 * x + 1.0 * y] =
# [22048, 24096, 32288, 52768]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
options)
self.assertEqual(weighted_host.weight, 11536)
self.assertEqual(weighted_host.host_state.host, 'host1')
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 52768)
self.assertEqual(weighed_host.obj.host, 'host4')
def test_weighted_sum_single_function(self):
fn_tuples = [(1.0, offset), ]
fns = ['nova.tests.scheduler.test_least_cost.compute_fake_weigher1']
self.flags(least_cost_functions=fns)
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=0
@ -80,24 +134,10 @@ class LeastCostTestCase(test.TestCase):
# [offset, ]=
# [10512, 11024, 13072, 18192]
# adjusted [ 2.0 * x ]=
# [21024, 22048, 26144, 36384]
# so, host1 should win:
options = {}
weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
options)
self.assertEqual(weighted_host.weight, 10512)
self.assertEqual(weighted_host.host_state.host, 'host1')
class TestWeightedHost(test.TestCase):
def test_dict_conversion_without_host_state(self):
host = least_cost.WeightedHost('someweight')
expected = {'weight': 'someweight'}
self.assertThat(host.to_dict(), matchers.DictMatches(expected))
def test_dict_conversion_with_host_state(self):
host_state = host_manager.HostState('somehost', None)
host = least_cost.WeightedHost('someweight', host_state)
expected = {'weight': 'someweight',
'host': 'somehost'}
self.assertThat(host.to_dict(), matchers.DictMatches(expected))
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 36384)
self.assertEqual(weighed_host.obj.host, 'host4')

117
nova/tests/scheduler/test_weights.py Normal file
View File

@ -0,0 +1,117 @@
# Copyright 2011-2012 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Tests For Scheduler weights.
"""
from nova import context
from nova.scheduler import weights
from nova import test
from nova.tests import matchers
from nova.tests.scheduler import fakes
class TestWeighedHost(test.TestCase):
def test_dict_conversion(self):
host_state = fakes.FakeHostState('somehost', None, {})
host = weights.WeighedHost(host_state, 'someweight')
expected = {'weight': 'someweight',
'host': 'somehost'}
self.assertThat(host.to_dict(), matchers.DictMatches(expected))
def test_all_weighers(self):
classes = weights.all_weighers()
class_names = [cls.__name__ for cls in classes]
self.assertEqual(len(classes), 1)
self.assertIn('RAMWeigher', class_names)
def test_all_weighers_with_deprecated_config1(self):
self.flags(compute_fill_first_cost_fn_weight=-1.0)
classes = weights.all_weighers()
class_names = [cls.__name__ for cls in classes]
self.assertEqual(len(classes), 1)
self.assertIn('_LeastCostWeigher', class_names)
def test_all_weighers_with_deprecated_config2(self):
self.flags(least_cost_functions=['something'])
classes = weights.all_weighers()
class_names = [cls.__name__ for cls in classes]
self.assertEqual(len(classes), 1)
self.assertIn('_LeastCostWeigher', class_names)
class RamWeigherTestCase(test.TestCase):
def setUp(self):
super(RamWeigherTestCase, self).setUp()
self.host_manager = fakes.FakeHostManager()
self.weight_handler = weights.HostWeightHandler()
self.weight_classes = self.weight_handler.get_matching_classes(
['nova.scheduler.weights.ram.RAMWeigher'])
def _get_weighed_host(self, hosts, weight_properties=None):
if weight_properties is None:
weight_properties = {}
return self.weight_handler.get_weighed_objects(self.weight_classes,
hosts, weight_properties)[0]
def _get_all_hosts(self):
ctxt = context.get_admin_context()
fakes.mox_host_manager_db_calls(self.mox, ctxt)
self.mox.ReplayAll()
host_states = self.host_manager.get_all_host_states(ctxt)
self.mox.VerifyAll()
self.mox.ResetAll()
return host_states
def test_default_of_spreading_first(self):
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=512
# host2: free_ram_mb=1024
# host3: free_ram_mb=3072
# host4: free_ram_mb=8192
# so, host4 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 8192)
self.assertEqual(weighed_host.obj.host, 'host4')
def test_ram_filter_multiplier1(self):
self.flags(ram_weight_multiplier=-1.0)
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=-512
# host2: free_ram_mb=-1024
# host3: free_ram_mb=-3072
# host4: free_ram_mb=-8192
# so, host1 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, -512)
self.assertEqual(weighed_host.obj.host, 'host1')
def test_ram_filter_multiplier2(self):
self.flags(ram_weight_multiplier=2.0)
hostinfo_list = self._get_all_hosts()
# host1: free_ram_mb=512 * 2
# host2: free_ram_mb=1024 * 2
# host3: free_ram_mb=3072 * 2
# host4: free_ram_mb=8192 * 2
# so, host4 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
self.assertEqual(weighed_host.weight, 8192 * 2)
self.assertEqual(weighed_host.obj.host, 'host4')

71
nova/weights.py Normal file
View File

@ -0,0 +1,71 @@
# Copyright (c) 2011-2012 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Pluggable Weighing support
"""
from nova import loadables
class WeighedObject(object):
"""Object with weight information."""
def __init__(self, obj, weight):
self.obj = obj
self.weight = weight
def __repr__(self):
return "<WeighedObject '%s': %s>" % (self.obj, self.weight)
class BaseWeigher(object):
"""Base class for pluggable weighers."""
def _weight_multiplier(self):
"""How weighted this weigher should be. Normally this would
be overriden in a subclass based on a config value.
"""
return 1.0
def _weigh_object(self, obj, weight_properties):
"""Override in a subclass to specify a weight for a specific
object.
"""
return 0.0
def weigh_objects(self, weighed_obj_list, weight_properties):
"""Weigh multiple objects. Override in a subclass if you need
need access to all objects in order to manipulate weights.
"""
for obj in weighed_obj_list:
obj.weight += (self._weight_multiplier() *
self._weigh_object(obj.obj, weight_properties))
class BaseWeightHandler(loadables.BaseLoader):
object_class = WeighedObject
def get_weighed_objects(self, weigher_classes, obj_list,
weighing_properties):
"""Return a sorted (highest score first) list of WeighedObjects."""
if not obj_list:
return []
weighed_objs = [self.object_class(obj, 0.0) for obj in obj_list]
for weigher_cls in weigher_classes:
weigher = weigher_cls()
weigher.weigh_objects(weighed_objs, weighing_properties)
return sorted(weighed_objs, key=lambda x: x.weight, reverse=True)