x/python-ganttclient
Code Issues Proposed changes

Fixing a bunch of conflicts

Browse Source
This commit is contained in:
Rick Harris
2011-06-01 00:54:33 +00:00
parent 92b087d4e6 f8f188a672
commit 39fb5aa5a9
3 changed files with 128 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
nova/scheduler/host_filter.py
View File

@@ -14,8 +14,8 @@
# under the License. # under the License.
""" """
Host Filter is a driver mechanism for requesting instance resources. Host Filter is a mechanism for requesting instance resources.
Three drivers are included: AllHosts, Flavor & JSON. AllHosts just Three filters are included: AllHosts, Flavor & JSON. AllHosts just
returns the full, unfiltered list of hosts. Flavor is a hard coded returns the full, unfiltered list of hosts. Flavor is a hard coded
matching mechanism based on flavor criteria and JSON is an ad-hoc matching mechanism based on flavor criteria and JSON is an ad-hoc
filter grammar. filter grammar.
@@ -43,17 +43,18 @@ from nova import flags
from nova import log as logging from nova import log as logging
from nova.scheduler import zone_aware_scheduler from nova.scheduler import zone_aware_scheduler
from nova import utils from nova import utils
from nova.scheduler import zone_aware_scheduler
LOG = logging.getLogger('nova.scheduler.host_filter') LOG = logging.getLogger('nova.scheduler.host_filter')
FLAGS = flags.FLAGS FLAGS = flags.FLAGS
flags.DEFINE_string('default_host_filter_driver', flags.DEFINE_string('default_host_filter',
'nova.scheduler.host_filter.AllHostsFilter', 'nova.scheduler.host_filter.AllHostsFilter',
'Which driver to use for filtering hosts.') 'Which filter to use for filtering hosts.')
class HostFilter(object): class HostFilter(object):
"""Base class for host filter drivers.""" """Base class for host filters."""
def instance_type_to_filter(self, instance_type): def instance_type_to_filter(self, instance_type):
"""Convert instance_type into a filter for most common use-case.""" """Convert instance_type into a filter for most common use-case."""
@@ -64,14 +65,15 @@ class HostFilter(object):
raise NotImplementedError() raise NotImplementedError()
def _full_name(self): def _full_name(self):
"""module.classname of the filter driver""" """module.classname of the filter."""
return "%s.%s" % (self.__module__, self.__class__.__name__) return "%s.%s" % (self.__module__, self.__class__.__name__)
class AllHostsFilter(HostFilter): class AllHostsFilter(HostFilter):
"""NOP host filter driver. Returns all hosts in ZoneManager. """ NOP host filter. Returns all hosts in ZoneManager.
This essentially does what the old Scheduler+Chance used This essentially does what the old Scheduler+Chance used
to give us.""" to give us.
"""
def instance_type_to_filter(self, instance_type): def instance_type_to_filter(self, instance_type):
"""Return anything to prevent base-class from raising """Return anything to prevent base-class from raising
@@ -85,7 +87,7 @@ class AllHostsFilter(HostFilter):
class InstanceTypeFilter(HostFilter): class InstanceTypeFilter(HostFilter):
"""HostFilter driver hard-coded to work with InstanceType records.""" """HostFilter hard-coded to work with InstanceType records."""
def instance_type_to_filter(self, instance_type): def instance_type_to_filter(self, instance_type):
"""Use instance_type to filter hosts.""" """Use instance_type to filter hosts."""
@@ -133,8 +135,9 @@ class InstanceTypeFilter(HostFilter):
class JsonFilter(HostFilter): class JsonFilter(HostFilter):
"""Host Filter driver to allow simple JSON-based grammar for """Host Filter to allow simple JSON-based grammar for
selecting hosts.""" selecting hosts.
"""
def _equals(self, args): def _equals(self, args):
"""First term is == all the other terms.""" """First term is == all the other terms."""
@@ -230,7 +233,8 @@ class JsonFilter(HostFilter):
def _parse_string(self, string, host, services): def _parse_string(self, string, host, services):
"""Strings prefixed with $ are capability lookups in the """Strings prefixed with $ are capability lookups in the
form '$service.capability[.subcap*]'""" form '$service.capability[.subcap*]'
"""
if not string: if not string:
return None return None
if string[0] != '$': if string[0] != '$':
@@ -273,43 +277,48 @@ class JsonFilter(HostFilter):
return hosts return hosts
DRIVERS = [AllHostsFilter, InstanceTypeFilter, JsonFilter] FILTERS = [AllHostsFilter, InstanceTypeFilter, JsonFilter]
def choose_driver(driver_name=None): def choose_host_filter(filter_name=None):
"""Since the caller may specify which driver to use we need """Since the caller may specify which filter to use we need
to have an authoritative list of what is permissible. This to have an authoritative list of what is permissible. This
function checks the driver name against a predefined set function checks the filter name against a predefined set
of acceptable drivers.""" of acceptable filters.
"""
if not driver_name: if not filter_name:
driver_name = FLAGS.default_host_filter_driver filter_name = FLAGS.default_host_filter
for filter_class in FILTERS:
try: host_match = "%s.%s" % (filter_class.__module__, filter_class.__name__)
driver = utils.import_object(driver_name) if host_match == filter_name:
return driver return filter_class()
except exception.ClassNotFound: raise exception.SchedulerHostFilterNotFound(filter_name=filter_name)
raise exception.SchedulerHostFilterDriverNotFound(
driver_name=driver_name)
class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler): class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler):
"""The HostFilterScheduler uses the HostFilter drivers to filter """The HostFilterScheduler uses the HostFilter to filter
hosts for weighing. The particular driver used may be passed in hosts for weighing. The particular filter used may be passed in
as an argument or the default will be used. as an argument or the default will be used.
request_spec = {'filter_driver': <Filter Driver name>, request_spec = {'filter': <Filter name>,
'instance_type': <InstanceType dict>} 'instance_type': <InstanceType dict>}
""" """
def filter_hosts(self, num, request_spec): def filter_hosts(self, num, request_spec):
"""Filter the full host list (from the ZoneManager)""" """Filter the full host list (from the ZoneManager)"""
driver_name = request_spec.get('filter_driver', None) filter_name = request_spec.get('filter', None)
driver = choose_driver(driver_name) host_filter = choose_host_filter(filter_name)
# TODO(sandy): We're only using InstanceType-based specs # TODO(sandy): We're only using InstanceType-based specs
# currently. Later we'll need to snoop for more detailed # currently. Later we'll need to snoop for more detailed
# host filter requests. # host filter requests.
instance_type = request_spec['instance_type'] instance_type = request_spec['instance_type']
name, query = driver.instance_type_to_filter(instance_type) name, query = host_filter.instance_type_to_filter(instance_type)
return driver.filter_hosts(self.zone_manager, query) return host_filter.filter_hosts(self.zone_manager, query)
def weigh_hosts(self, num, request_spec, hosts):
"""Derived classes must override this method and return
a lists of hosts in [{weight, hostname}] format.
"""
return [dict(weight=1, hostname=host) for host, caps in hosts]

19
nova/scheduler/zone_aware_scheduler.py
View File

@@ -23,8 +23,8 @@ across zones. There are two expansion points to this class for:
import operator import operator
from nova import db from nova import db
from nova import rpc
from nova import log as logging from nova import log as logging
from nova import rpc
from nova.scheduler import api from nova.scheduler import api
from nova.scheduler import driver from nova.scheduler import driver
@@ -43,12 +43,11 @@ class ZoneAwareScheduler(driver.Scheduler):
"""This method is called from nova.compute.api to provision """This method is called from nova.compute.api to provision
an instance. However we need to look at the parameters being an instance. However we need to look at the parameters being
passed in to see if this is a request to: passed in to see if this is a request to:
1. Create a Build Plan and then provision, or 1. Create a Build Plan and then provision, or
2. Use the Build Plan information in the request parameters 2. Use the Build Plan information in the request parameters
to simply create the instance (either in this zone or to simply create the instance (either in this zone or
a child zone).""" a child zone).
"""
# TODO(sandy): We'll have to look for richer specs at some point. # TODO(sandy): We'll have to look for richer specs at some point.
@@ -81,15 +80,15 @@ class ZoneAwareScheduler(driver.Scheduler):
% locals()) % locals())
else: else:
# TODO(sandy) Provision in child zone ... # TODO(sandy) Provision in child zone ...
LOG.warning(_("Provision to Child Zone not supported (yet)") LOG.warning(_("Provision to Child Zone not supported (yet)"))
% locals())
pass pass
def select(self, context, request_spec, *args, **kwargs): def select(self, context, request_spec, *args, **kwargs):
"""Select returns a list of weights and zone/host information """Select returns a list of weights and zone/host information
corresponding to the best hosts to service the request. Any corresponding to the best hosts to service the request. Any
child zone information has been encrypted so as not to reveal child zone information has been encrypted so as not to reveal
anything about the children.""" anything about the children.
"""
return self._schedule(context, "compute", request_spec, return self._schedule(context, "compute", request_spec,
*args, **kwargs) *args, **kwargs)
@@ -143,7 +142,8 @@ class ZoneAwareScheduler(driver.Scheduler):
def filter_hosts(self, num, request_spec): def filter_hosts(self, num, request_spec):
"""Derived classes must override this method and return """Derived classes must override this method and return
a list of hosts in [(hostname, capability_dict)] format.""" a list of hosts in [(hostname, capability_dict)] format.
"""
# NOTE(sirp): The default logic is the equivalent to AllHostsFilter # NOTE(sirp): The default logic is the equivalent to AllHostsFilter
service_states = self.zone_manager.service_states service_states = self.zone_manager.service_states
return [(host, services) return [(host, services)
@@ -151,6 +151,7 @@ class ZoneAwareScheduler(driver.Scheduler):
def weigh_hosts(self, num, request_spec, hosts): def weigh_hosts(self, num, request_spec, hosts):
"""Derived classes may override this to provide more sophisticated """Derived classes may override this to provide more sophisticated
scheduling objectives""" scheduling objectives
"""
# NOTE(sirp): The default logic is the same as the NoopCostFunction # NOTE(sirp): The default logic is the same as the NoopCostFunction
return [dict(weight=1, hostname=host) for host, caps in hosts] return [dict(weight=1, hostname=host) for host, caps in hosts]

132
nova/tests/scheduler/test_host_filter.py
View File

@@ -13,7 +13,7 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
""" """
Tests For Scheduler Host Filter Drivers. Tests For Scheduler Host Filters.
""" """
import json import json
@@ -22,18 +22,43 @@ from nova import exception
from nova import flags from nova import flags
from nova import test from nova import test
from nova.scheduler import host_filter from nova.scheduler import host_filter
from nova.tests.scheduler import test_zone_aware_scheduler
FLAGS = flags.FLAGS FLAGS = flags.FLAGS
class FakeZoneManager:
pass
class HostFilterTestCase(test.TestCase): class HostFilterTestCase(test.TestCase):
"""Test case for host filter drivers.""" """Test case for host filters."""
def _host_caps(self, multiplier):
# Returns host capabilities in the following way:
# host1 = memory:free 10 (100max)
# disk:available 100 (1000max)
# hostN = memory:free 10 + 10N
# disk:available 100 + 100N
# in other words: hostN has more resources than host0
# which means ... don't go above 10 hosts.
return {'host_name-description': 'XenServer %s' % multiplier,
'host_hostname': 'xs-%s' % multiplier,
'host_memory_total': 100,
'host_memory_overhead': 10,
'host_memory_free': 10 + multiplier * 10,
'host_memory_free-computed': 10 + multiplier * 10,
'host_other-config': {},
'host_ip_address': '192.168.1.%d' % (100 + multiplier),
'host_cpu_info': {},
'disk_available': 100 + multiplier * 100,
'disk_total': 1000,
'disk_used': 0,
'host_uuid': 'xxx-%d' % multiplier,
'host_name-label': 'xs-%s' % multiplier}
def setUp(self): def setUp(self):
super(HostFilterTestCase, self).setUp() self.old_flag = FLAGS.default_host_filter
self.old_flag = FLAGS.default_host_filter_driver FLAGS.default_host_filter = \
FLAGS.default_host_filter_driver = \
'nova.scheduler.host_filter.AllHostsFilter' 'nova.scheduler.host_filter.AllHostsFilter'
self.instance_type = dict(name='tiny', self.instance_type = dict(name='tiny',
memory_mb=50, memory_mb=50,
@@ -44,63 +69,59 @@ class HostFilterTestCase(test.TestCase):
rxtx_quota=30000, rxtx_quota=30000,
rxtx_cap=200) rxtx_cap=200)
class FakeZoneManager:
pass
self.zone_manager = FakeZoneManager() self.zone_manager = FakeZoneManager()
states = {}
states = test_zone_aware_scheduler.fake_zone_manager_service_states( for x in xrange(10):
num_hosts=10) states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)}
self.zone_manager.service_states = states self.zone_manager.service_states = states
def tearDown(self): def tearDown(self):
FLAGS.default_host_filter_driver = self.old_flag FLAGS.default_host_filter = self.old_flag
super(HostFilterTestCase, self).tearDown()
def test_choose_driver(self): def test_choose_filter(self):
# Test default driver ... # Test default filter ...
driver = host_filter.choose_driver() hf = host_filter.choose_host_filter()
self.assertEquals(driver._full_name(), self.assertEquals(hf._full_name(),
'nova.scheduler.host_filter.AllHostsFilter') 'nova.scheduler.host_filter.AllHostsFilter')
# Test valid driver ... # Test valid filter ...
driver = host_filter.choose_driver( hf = host_filter.choose_host_filter(
'nova.scheduler.host_filter.InstanceTypeFilter') 'nova.scheduler.host_filter.InstanceTypeFilter')
self.assertEquals(driver._full_name(), self.assertEquals(hf._full_name(),
'nova.scheduler.host_filter.InstanceTypeFilter') 'nova.scheduler.host_filter.InstanceTypeFilter')
# Test invalid driver ... # Test invalid filter ...
try: try:
host_filter.choose_driver('does not exist') host_filter.choose_host_filter('does not exist')
self.fail("Should not find driver") self.fail("Should not find host filter.")
except exception.SchedulerHostFilterDriverNotFound: except exception.SchedulerHostFilterNotFound:
pass pass
def test_all_host_driver(self): def test_all_host_filter(self):
driver = host_filter.AllHostsFilter() hf = host_filter.AllHostsFilter()
cooked = driver.instance_type_to_filter(self.instance_type) cooked = hf.instance_type_to_filter(self.instance_type)
hosts = driver.filter_hosts(self.zone_manager, cooked) hosts = hf.filter_hosts(self.zone_manager, cooked)
self.assertEquals(10, len(hosts)) self.assertEquals(10, len(hosts))
for host, capabilities in hosts: for host, capabilities in hosts:
self.assertTrue(host.startswith('host')) self.assertTrue(host.startswith('host'))
def test_instance_type_driver(self): def test_instance_type_filter(self):
driver = host_filter.InstanceTypeFilter() hf = host_filter.InstanceTypeFilter()
# filter all hosts that can support 50 ram and 500 disk # filter all hosts that can support 50 ram and 500 disk
name, cooked = driver.instance_type_to_filter(self.instance_type) name, cooked = hf.instance_type_to_filter(self.instance_type)
self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter',
name) name)
hosts = driver.filter_hosts(self.zone_manager, cooked) hosts = hf.filter_hosts(self.zone_manager, cooked)
self.assertEquals(6, len(hosts)) self.assertEquals(6, len(hosts))
just_hosts = [host for host, caps in hosts] just_hosts = [host for host, caps in hosts]
just_hosts.sort() just_hosts.sort()
self.assertEquals('host05', just_hosts[0]) self.assertEquals('host05', just_hosts[0])
self.assertEquals('host10', just_hosts[5]) self.assertEquals('host10', just_hosts[5])
def test_json_driver(self): def test_json_filter(self):
driver = host_filter.JsonFilter() hf = host_filter.JsonFilter()
# filter all hosts that can support 50 ram and 500 disk # filter all hosts that can support 50 ram and 500 disk
name, cooked = driver.instance_type_to_filter(self.instance_type) name, cooked = hf.instance_type_to_filter(self.instance_type)
self.assertEquals('nova.scheduler.host_filter.JsonFilter', name) self.assertEquals('nova.scheduler.host_filter.JsonFilter', name)
hosts = driver.filter_hosts(self.zone_manager, cooked) hosts = hf.filter_hosts(self.zone_manager, cooked)
self.assertEquals(6, len(hosts)) self.assertEquals(6, len(hosts))
just_hosts = [host for host, caps in hosts] just_hosts = [host for host, caps in hosts]
just_hosts.sort() just_hosts.sort()
@@ -120,7 +141,7 @@ class HostFilterTestCase(test.TestCase):
] ]
] ]
cooked = json.dumps(raw) cooked = json.dumps(raw)
hosts = driver.filter_hosts(self.zone_manager, cooked) hosts = hf.filter_hosts(self.zone_manager, cooked)
self.assertEquals(5, len(hosts)) self.assertEquals(5, len(hosts))
just_hosts = [host for host, caps in hosts] just_hosts = [host for host, caps in hosts]
@@ -132,7 +153,7 @@ class HostFilterTestCase(test.TestCase):
['=', '$compute.host_memory_free', 30], ['=', '$compute.host_memory_free', 30],
] ]
cooked = json.dumps(raw) cooked = json.dumps(raw)
hosts = driver.filter_hosts(self.zone_manager, cooked) hosts = hf.filter_hosts(self.zone_manager, cooked)
self.assertEquals(9, len(hosts)) self.assertEquals(9, len(hosts))
just_hosts = [host for host, caps in hosts] just_hosts = [host for host, caps in hosts]
@@ -142,7 +163,7 @@ class HostFilterTestCase(test.TestCase):
raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100]
cooked = json.dumps(raw) cooked = json.dumps(raw)
hosts = driver.filter_hosts(self.zone_manager, cooked) hosts = hf.filter_hosts(self.zone_manager, cooked)
self.assertEquals(5, len(hosts)) self.assertEquals(5, len(hosts))
just_hosts = [host for host, caps in hosts] just_hosts = [host for host, caps in hosts]
@@ -154,35 +175,32 @@ class HostFilterTestCase(test.TestCase):
raw = ['unknown command', ] raw = ['unknown command', ]
cooked = json.dumps(raw) cooked = json.dumps(raw)
try: try:
driver.filter_hosts(self.zone_manager, cooked) hf.filter_hosts(self.zone_manager, cooked)
self.fail("Should give KeyError") self.fail("Should give KeyError")
except KeyError, e: except KeyError, e:
pass pass
self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([]))) self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps([])))
self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({}))) self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps({})))
self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps( self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps(
['not', True, False, True, False] ['not', True, False, True, False]
))) )))
try: try:
driver.filter_hosts(self.zone_manager, json.dumps( hf.filter_hosts(self.zone_manager, json.dumps(
'not', True, False, True, False 'not', True, False, True, False
)) ))
self.fail("Should give KeyError") self.fail("Should give KeyError")
except KeyError, e: except KeyError, e:
pass pass
self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( self.assertFalse(hf.filter_hosts(self.zone_manager,
['=', '$foo', 100] json.dumps(['=', '$foo', 100])))
))) self.assertFalse(hf.filter_hosts(self.zone_manager,
self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( json.dumps(['=', '$.....', 100])))
['=', '$.....', 100] self.assertFalse(hf.filter_hosts(self.zone_manager,
))) json.dumps(
self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]])))
['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]]
)))
self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( self.assertFalse(hf.filter_hosts(self.zone_manager,
['=', {}, ['>', '$missing....foo']] json.dumps(['=', {}, ['>', '$missing....foo']])))
)))