x/python-ganttclient
Code Issues Proposed changes

Refactored the HostFilterScheduler and LeastCostScheduler classes so that they can be combined into a single class that can do both host filtering and host weighting, allowing subclasses to override those processes as needed. Also renamed the ZoneAwareScheduler to AbstractScheduler, for two reasons: one, the 'zone-aware' designation was necessary when the zone code was being developed; now that it is part of nova, it is not an important distinction. Second, the 'Abstract' part clearly indicates that this is a class that is not designed to be used directly, but rather as the basis for specific scheduler subclasses.

Browse Source
This commit is contained in:
Ed Leafe
2011-08-16 13:01:41 +00:00
committed by Tarmac
parent 3ed5116b3f 6edce0fcf5
commit c7c5f341bb
5 changed files with 61 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
nova/scheduler/zone_aware_scheduler.py → nova/scheduler/abstract_scheduler.py
View File

@@ -14,7 +14,7 @@
# under the License.
"""
The Zone Aware Scheduler is a base class Scheduler for creating instances
The AbsractScheduler is a base class Scheduler for creating instances
across zones. There are two expansion points to this class for:
1. Assigning Weights to hosts for requested instances
2. Filtering Hosts based on required instance capabilities
@@ -40,7 +40,7 @@ from nova.scheduler import api
from nova.scheduler import driver
FLAGS = flags.FLAGS
LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler')
LOG = logging.getLogger('nova.scheduler.abstract_scheduler')
class InvalidBlob(exception.NovaException):
@@ -48,8 +48,10 @@ class InvalidBlob(exception.NovaException):
"to instance create request.")
class ZoneAwareScheduler(driver.Scheduler):
"""Base class for creating Zone Aware Schedulers."""
class AbstractScheduler(driver.Scheduler):
"""Base class for creating Schedulers that can work across any nova
deployment, from simple designs to multiply-nested zones.
"""
def _call_zone_method(self, context, method, specs, zones):
"""Call novaclient zone method. Broken out for testing."""
@@ -266,7 +268,7 @@ class ZoneAwareScheduler(driver.Scheduler):
"""
if topic != "compute":
raise NotImplementedError(_("Zone Aware Scheduler only understands"
raise NotImplementedError(_("Scheduler only understands"
" Compute nodes (for now)"))
num_instances = request_spec.get('num_instances', 1)
@@ -328,13 +330,31 @@ class ZoneAwareScheduler(driver.Scheduler):
requested_mem = instance_type['memory_mb'] * 1024 * 1024
return capabilities['host_memory_free'] >= requested_mem
def hold_filter_hosts(self, topic, request_spec, hosts=None):
"""Filter the full host list (from the ZoneManager)"""
# NOTE(dabo): The logic used by the current _schedule() method
# is incorrect. Since this task is just to refactor the classes,
# I'm not fixing the logic now - that will be the next task.
# So for now this method is just renamed; afterwards this will
# become the filter_hosts() method, and the one below will
# be removed.
filter_name = request_spec.get('filter', None)
# Make sure that the requested filter is legitimate.
selected_filter = host_filter.choose_host_filter(filter_name)
# TODO(sandy): We're only using InstanceType-based specs
# currently. Later we'll need to snoop for more detailed
# host filter requests.
instance_type = request_spec['instance_type']
name, query = selected_filter.instance_type_to_filter(instance_type)
return selected_filter.filter_hosts(self.zone_manager, query)
def filter_hosts(self, topic, request_spec, host_list=None):
"""Return a list of hosts which are acceptable for scheduling.
Return value should be a list of (hostname, capability_dict)s.
Derived classes may override this, but may find the
'<topic>_filter' function more appropriate.
"""
def _default_filter(self, hostname, capabilities, request_spec):
"""Default filter function if there's no <topic>_filter"""
# NOTE(sirp): The default logic is the equivalent to

51
nova/scheduler/host_filter.py
View File

@@ -14,7 +14,12 @@
# under the License.
"""
Host Filter is a mechanism for requesting instance resources.
The Host Filter classes are a way to ensure that only hosts that are
appropriate are considered when creating a new instance. Hosts that are
either incompatible or insufficient to accept a newly-requested instance
are removed by Host Filter classes from consideration. Those that pass
the filter are then passed on for weighting or other process for ordering.
Three filters are included: AllHosts, Flavor & JSON. AllHosts just
returns the full, unfiltered list of hosts. Flavor is a hard coded
matching mechanism based on flavor criteria and JSON is an ad-hoc
@@ -28,12 +33,6 @@ noted a need for a more expressive way of specifying instances.
Since we don't want to get into building full DSL this is a simple
form as an example of how this could be done. In reality, most
consumers will use the more rigid filters such as FlavorFilter.
Note: These are "required" capability filters. These capabilities
used must be present or the host will be excluded. The hosts
returned are then weighed by the Weighted Scheduler. Weights
can take the more esoteric factors into consideration (such as
server affinity and customer separation).
"""
import json
@@ -41,9 +40,7 @@ import json
from nova import exception
from nova import flags
from nova import log as logging
from nova.scheduler import zone_aware_scheduler
from nova import utils
from nova.scheduler import zone_aware_scheduler
LOG = logging.getLogger('nova.scheduler.host_filter')
@@ -125,9 +122,8 @@ class InstanceTypeFilter(HostFilter):
spec_disk = instance_type['local_gb']
extra_specs = instance_type['extra_specs']
if host_ram_mb >= spec_ram and \
disk_bytes >= spec_disk and \
self._satisfies_extra_specs(capabilities, instance_type):
if ((host_ram_mb >= spec_ram) and (disk_bytes >= spec_disk) and
self._satisfies_extra_specs(capabilities, instance_type)):
selected_hosts.append((host, capabilities))
return selected_hosts
@@ -309,7 +305,6 @@ def choose_host_filter(filter_name=None):
function checks the filter name against a predefined set
of acceptable filters.
"""
if not filter_name:
filter_name = FLAGS.default_host_filter
for filter_class in FILTERS:
@@ -317,33 +312,3 @@ def choose_host_filter(filter_name=None):
if host_match == filter_name:
return filter_class()
raise exception.SchedulerHostFilterNotFound(filter_name=filter_name)
class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler):
"""The HostFilterScheduler uses the HostFilter to filter
hosts for weighing. The particular filter used may be passed in
as an argument or the default will be used.
request_spec = {'filter': <Filter name>,
'instance_type': <InstanceType dict>}
"""
def filter_hosts(self, topic, request_spec, hosts=None):
"""Filter the full host list (from the ZoneManager)"""
filter_name = request_spec.get('filter', None)
host_filter = choose_host_filter(filter_name)
# TODO(sandy): We're only using InstanceType-based specs
# currently. Later we'll need to snoop for more detailed
# host filter requests.
instance_type = request_spec['instance_type']
name, query = host_filter.instance_type_to_filter(instance_type)
return host_filter.filter_hosts(self.zone_manager, query)
def weigh_hosts(self, topic, request_spec, hosts):
"""Derived classes must override this method and return
a lists of hosts in [{weight, hostname}] format.
"""
return [dict(weight=1, hostname=hostname, capabilities=caps)
for hostname, caps in hosts]

7
nova/scheduler/least_cost.py
View File

@@ -22,11 +22,14 @@ The cost-function and weights are tabulated, and the host with the least cost
is then selected for provisioning.
"""
# TODO(dabo): This class will be removed in the next merge prop; it remains now
# because much of the code will be refactored into different classes.
import collections
from nova import flags
from nova import log as logging
from nova.scheduler import zone_aware_scheduler
from nova.scheduler import abstract_scheduler
from nova import utils
from nova import exception
@@ -61,7 +64,7 @@ def compute_fill_first_cost_fn(host):
return free_mem
class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler):
class LeastCostScheduler(abstract_scheduler.AbstractScheduler):
def __init__(self, *args, **kwargs):
self.cost_fns_cache = {}
super(LeastCostScheduler, self).__init__(*args, **kwargs)

40
nova/tests/scheduler/test_zone_aware_scheduler.py → nova/tests/scheduler/test_abstract_scheduler.py
View File

@@ -13,7 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
"""
Tests For Zone Aware Scheduler.
Tests For Abstract Scheduler.
"""
import json
@@ -25,7 +25,7 @@ from nova import rpc
from nova import test
from nova.compute import api as compute_api
from nova.scheduler import driver
from nova.scheduler import zone_aware_scheduler
from nova.scheduler import abstract_scheduler
from nova.scheduler import zone_manager
@@ -60,7 +60,7 @@ def fake_zone_manager_service_states(num_hosts):
return states
class FakeZoneAwareScheduler(zone_aware_scheduler.ZoneAwareScheduler):
class FakeAbstractScheduler(abstract_scheduler.AbstractScheduler):
# No need to stub anything at the moment
pass
@@ -161,15 +161,15 @@ def fake_zone_get_all(context):
]
class ZoneAwareSchedulerTestCase(test.TestCase):
"""Test case for Zone Aware Scheduler."""
class AbstractSchedulerTestCase(test.TestCase):
"""Test case for Abstract Scheduler."""
def test_zone_aware_scheduler(self):
def test_abstract_scheduler(self):
"""
Create a nested set of FakeZones, try to build multiple instances
and ensure that a select call returns the appropriate build plan.
"""
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method)
self.stubs.Set(nova.db, 'zone_get_all', fake_zone_get_all)
@@ -194,7 +194,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
properly adjusted based on the scale/offset in the zone
db entries.
"""
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
child_results = fake_call_zone_method(None, None, None, None)
zones = fake_zone_get_all(None)
sched._adjust_child_weights(child_results, zones)
@@ -209,11 +209,11 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
if zone == 'zone3': # Scale x1000
self.assertEqual(scaled.pop(0), w)
def test_empty_zone_aware_scheduler(self):
def test_empty_abstract_scheduler(self):
"""
Ensure empty hosts & child_zones result in NoValidHosts exception.
"""
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
self.stubs.Set(sched, '_call_zone_method', fake_empty_call_zone_method)
self.stubs.Set(nova.db, 'zone_get_all', fake_zone_get_all)
@@ -231,7 +231,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
If the zone_blob hint was passed in, don't re-schedule.
"""
global was_called
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
was_called = False
self.stubs.Set(sched, '_provision_resource', fake_provision_resource)
request_spec = {
@@ -248,7 +248,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
def test_provision_resource_local(self):
"""Provision a resource locally or remotely."""
global was_called
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
was_called = False
self.stubs.Set(sched, '_provision_resource_locally',
fake_provision_resource_locally)
@@ -260,7 +260,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
def test_provision_resource_remote(self):
"""Provision a resource locally or remotely."""
global was_called
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
was_called = False
self.stubs.Set(sched, '_provision_resource_from_blob',
fake_provision_resource_from_blob)
@@ -272,9 +272,9 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
def test_provision_resource_from_blob_empty(self):
"""Provision a resource locally or remotely given no hints."""
global was_called
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
request_spec = {}
self.assertRaises(zone_aware_scheduler.InvalidBlob,
self.assertRaises(abstract_scheduler.InvalidBlob,
sched._provision_resource_from_blob,
None, {}, 1, {}, {})
@@ -283,7 +283,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
Provision a resource locally or remotely when blob hint passed in.
"""
global was_called
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
was_called = False
def fake_create_db_entry_for_new_instance(self, context,
@@ -317,7 +317,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
passed in.
"""
global was_called
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
self.stubs.Set(sched, '_decrypt_blob',
fake_decrypt_blob_returns_child_info)
was_called = False
@@ -336,7 +336,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
from an immediate child.
"""
global was_called
sched = FakeZoneAwareScheduler()
sched = FakeAbstractScheduler()
was_called = False
self.stubs.Set(sched, '_ask_child_zone_to_create_instance',
fake_ask_child_zone_to_create_instance)
@@ -350,14 +350,14 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
def test_decrypt_blob(self):
"""Test that the decrypt method works."""
fixture = FakeZoneAwareScheduler()
fixture = FakeAbstractScheduler()
test_data = {"foo": "bar"}
class StubDecryptor(object):
def decryptor(self, key):
return lambda blob: blob
self.stubs.Set(zone_aware_scheduler, 'crypto',
self.stubs.Set(abstract_scheduler, 'crypto',
StubDecryptor())
self.assertEqual(fixture._decrypt_blob(test_data),

4
nova/tests/scheduler/test_least_cost_scheduler.py
View File

@@ -18,7 +18,7 @@ Tests For Least Cost Scheduler
from nova import test
from nova.scheduler import least_cost
from nova.tests.scheduler import test_zone_aware_scheduler
from nova.tests.scheduler import test_abstract_scheduler
MB = 1024 * 1024
@@ -70,7 +70,7 @@ class LeastCostSchedulerTestCase(test.TestCase):
zone_manager = FakeZoneManager()
states = test_zone_aware_scheduler.fake_zone_manager_service_states(
states = test_abstract_scheduler.fake_zone_manager_service_states(
num_hosts=10)
zone_manager.service_states = states