From 43697e7dea0c1ffb3c0ebbb7b955bf511705bb0b Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 12 Aug 2011 16:19:46 -0500 Subject: [PATCH] end of day --- nova/scheduler/filters/__init__.py | 1 + nova/scheduler/filters/abstract_filter.py | 54 +------------ nova/scheduler/filters/all_hosts_filter.py | 3 +- .../scheduler/filters/instance_type_filter.py | 5 +- nova/scheduler/filters/json_filter.py | 39 +++++++--- nova/scheduler/host_filter.py | 75 +++++++++++++++++++ nova/tests/scheduler/test_host_filter.py | 34 +++++---- 7 files changed, 129 insertions(+), 82 deletions(-) create mode 100644 nova/scheduler/host_filter.py diff --git a/nova/scheduler/filters/__init__.py b/nova/scheduler/filters/__init__.py index 27160ca0a..4c9187c5a 100644 --- a/nova/scheduler/filters/__init__.py +++ b/nova/scheduler/filters/__init__.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +from abstract_filter import AbstractHostFilter from all_hosts_filter import AllHostsFilter from instance_type_filter import InstanceTypeFilter from json_filter import JsonFilter diff --git a/nova/scheduler/filters/abstract_filter.py b/nova/scheduler/filters/abstract_filter.py index 05982820f..fe5610923 100644 --- a/nova/scheduler/filters/abstract_filter.py +++ b/nova/scheduler/filters/abstract_filter.py @@ -13,44 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. -""" -The Host Filter classes are a way to ensure that only hosts that are -appropriate are considered when creating a new instance. Hosts that are -either incompatible or insufficient to accept a newly-requested instance -are removed by Host Filter classes from consideration. Those that pass -the filter are then passed on for weighting or other process for ordering. - -Three filters are included: AllHosts, Flavor & JSON. AllHosts just -returns the full, unfiltered list of hosts. Flavor is a hard coded -matching mechanism based on flavor criteria and JSON is an ad-hoc -filter grammar. - -Why JSON? The requests for instances may come in through the -REST interface from a user or a parent Zone. -Currently Flavors and/or InstanceTypes are used for -specifing the type of instance desired. Specific Nova users have -noted a need for a more expressive way of specifying instances. -Since we don't want to get into building full DSL this is a simple -form as an example of how this could be done. In reality, most -consumers will use the more rigid filters such as FlavorFilter. -""" - -import json - -from nova import exception -from nova import flags -from nova import log as logging import nova.scheduler +from nova import flags - -LOG = logging.getLogger('nova.scheduler.host_filter') FLAGS = flags.FLAGS flags.DEFINE_string('default_host_filter', - 'nova.scheduler.host_filter.AllHostsFilter', + 'nova.scheduler.filters.AllHostsFilter', 'Which filter to use for filtering hosts') - class AbstractHostFilter(object): """Base class for host filters.""" def instance_type_to_filter(self, instance_type): @@ -64,24 +35,3 @@ class AbstractHostFilter(object): def _full_name(self): """module.classname of the filter.""" return "%s.%s" % (self.__module__, self.__class__.__name__) - - -def _get_filters(): - from nova.scheduler import filters - return [itm for itm in dir(filters) - if issubclass(itm, AbstractHostFilter)] - - -def choose_host_filter(filter_name=None): - """Since the caller may specify which filter to use we need - to have an authoritative list of what is permissible. This - function checks the filter name against a predefined set - of acceptable filters. - """ - if not filter_name: - filter_name = FLAGS.default_host_filter - for filter_class in _get_filters(): - host_match = "%s.%s" % (filter_class.__module__, filter_class.__name__) - if host_match == filter_name: - return filter_class() - raise exception.SchedulerHostFilterNotFound(filter_name=filter_name) diff --git a/nova/scheduler/filters/all_hosts_filter.py b/nova/scheduler/filters/all_hosts_filter.py index bc4acfd1a..e80d829ca 100644 --- a/nova/scheduler/filters/all_hosts_filter.py +++ b/nova/scheduler/filters/all_hosts_filter.py @@ -15,9 +15,10 @@ import nova.scheduler +from nova.scheduler.filters import abstract_filter -class AllHostsFilter(nova.scheduler.host_filter.AbstractHostFilter): +class AllHostsFilter(abstract_filter.AbstractHostFilter): """NOP host filter. Returns all hosts in ZoneManager.""" def instance_type_to_filter(self, instance_type): """Return anything to prevent base-class from raising diff --git a/nova/scheduler/filters/instance_type_filter.py b/nova/scheduler/filters/instance_type_filter.py index 03ffc46c6..62b9ee414 100644 --- a/nova/scheduler/filters/instance_type_filter.py +++ b/nova/scheduler/filters/instance_type_filter.py @@ -14,10 +14,11 @@ # under the License. -from nova.scheduler import host_filter +import nova.scheduler +from nova.scheduler.filters import abstract_filter -class InstanceTypeFilter(host_filter.AbstractHostFilter): +class InstanceTypeFilter(abstract_filter.AbstractHostFilter): """HostFilter hard-coded to work with InstanceType records.""" def instance_type_to_filter(self, instance_type): """Use instance_type to filter hosts.""" diff --git a/nova/scheduler/filters/json_filter.py b/nova/scheduler/filters/json_filter.py index 358abdc4d..889b96915 100644 --- a/nova/scheduler/filters/json_filter.py +++ b/nova/scheduler/filters/json_filter.py @@ -14,49 +14,64 @@ # under the License. +import json import operator -from nova.scheduler import host_filter +import nova.scheduler +from nova.scheduler.filters import abstract_filter + +def debug(*args): + with file("/tmp/debug", "a") as dbg: + msg = " ".join([str(arg) for arg in args]) + dbg.write("%s\n" % msg) -class JsonFilter(host_filter.AbstractHostFilter): +class JsonFilter(abstract_filter.AbstractHostFilter): """Host Filter to allow simple JSON-based grammar for selecting hosts. """ - def _op_comp(self, args, op): + def _op_compare(self, args, op): """Returns True if the specified operator can successfully compare the first item in the args with all the rest. Will return False if only one item is in the list. """ if len(args) < 2: return False - bad = [arg for arg in args[1:] - if not op(args[0], arg)] + if op is operator.contains: + debug("ARGS", type(args), args) + debug("op", op) + debug("REVERSED!!!") + # operator.contains reverses the param order. + bad = [arg for arg in args[1:] + if not op(args, args[0])] + else: + bad = [arg for arg in args[1:] + if not op(args[0], arg)] return not bool(bad) def _equals(self, args): """First term is == all the other terms.""" - return self._op_comp(args, operator.eq) + return self._op_compare(args, operator.eq) def _less_than(self, args): """First term is < all the other terms.""" - return self._op_comp(args, operator.lt) + return self._op_compare(args, operator.lt) def _greater_than(self, args): """First term is > all the other terms.""" - return self._op_comp(args, operator.gt) + return self._op_compare(args, operator.gt) def _in(self, args): """First term is in set of remaining terms""" - return self._op_comp(args, operator.contains) + return self._op_compare(args, operator.contains) def _less_than_equal(self, args): """First term is <= all the other terms.""" - return self._op_comp(args, operator.le) + return self._op_compare(args, operator.le) def _greater_than_equal(self, args): """First term is >= all the other terms.""" - return self._op_comp(args, operator.ge) + return self._op_compare(args, operator.ge) def _not(self, args): """Flip each of the arguments.""" @@ -129,6 +144,8 @@ class JsonFilter(host_filter.AbstractHostFilter): specified in the query. """ expanded = json.loads(query) + + debug("expanded", type(expanded), expanded) filtered_hosts = [] for host, services in zone_manager.service_states.iteritems(): result = self._process_filter(zone_manager, expanded, host, diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py new file mode 100644 index 000000000..f5191f5c9 --- /dev/null +++ b/nova/scheduler/host_filter.py @@ -0,0 +1,75 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +The Host Filter classes are a way to ensure that only hosts that are +appropriate are considered when creating a new instance. Hosts that are +either incompatible or insufficient to accept a newly-requested instance +are removed by Host Filter classes from consideration. Those that pass +the filter are then passed on for weighting or other process for ordering. + +Three filters are included: AllHosts, Flavor & JSON. AllHosts just +returns the full, unfiltered list of hosts. Flavor is a hard coded +matching mechanism based on flavor criteria and JSON is an ad-hoc +filter grammar. + +Why JSON? The requests for instances may come in through the +REST interface from a user or a parent Zone. +Currently Flavors and/or InstanceTypes are used for +specifing the type of instance desired. Specific Nova users have +noted a need for a more expressive way of specifying instances. +Since we don't want to get into building full DSL this is a simple +form as an example of how this could be done. In reality, most +consumers will use the more rigid filters such as FlavorFilter. +""" + +import json +import types + +from nova import exception +from nova import flags +from nova import log as logging + +import nova.scheduler + + +LOG = logging.getLogger('nova.scheduler.host_filter') +FLAGS = flags.FLAGS + + +def _get_filters(): + from nova.scheduler import filters + def get_itm(nm): + return getattr(filters, nm) + + return [get_itm(itm) for itm in dir(filters) + if (type(get_itm(itm)) is types.TypeType) + and issubclass(get_itm(itm), filters.AbstractHostFilter)] + + +def choose_host_filter(filter_name=None): + """Since the caller may specify which filter to use we need + to have an authoritative list of what is permissible. This + function checks the filter name against a predefined set + of acceptable filters. + """ + if not filter_name: + filter_name = FLAGS.default_host_filter + for filter_class in _get_filters(): + host_match = "%s.%s" % (filter_class.__module__, filter_class.__name__) + if (host_match.startswith("nova.scheduler.filters") and + (host_match.split(".")[-1] == filter_name)): + return filter_class() + raise exception.SchedulerHostFilterNotFound(filter_name=filter_name) diff --git a/nova/tests/scheduler/test_host_filter.py b/nova/tests/scheduler/test_host_filter.py index 818be2f45..a64b25138 100644 --- a/nova/tests/scheduler/test_host_filter.py +++ b/nova/tests/scheduler/test_host_filter.py @@ -20,6 +20,7 @@ import json from nova import exception from nova import test +from nova.scheduler import host_filter from nova.scheduler import filters @@ -55,7 +56,7 @@ class HostFilterTestCase(test.TestCase): def setUp(self): super(HostFilterTestCase, self).setUp() - default_host_filter = 'nova.scheduler.filteris.AllHostsFilter' + default_host_filter = 'AllHostsFilter' self.flags(default_host_filter=default_host_filter) self.instance_type = dict(name='tiny', memory_mb=50, @@ -98,13 +99,10 @@ class HostFilterTestCase(test.TestCase): def test_choose_filter(self): # Test default filter ... hf = host_filter.choose_host_filter() - self.assertEquals(hf._full_name(), - 'nova.scheduler.host_filter.AllHostsFilter') + self.assertEquals(hf._full_name().split(".")[-1], 'AllHostsFilter') # Test valid filter ... - hf = host_filter.choose_host_filter( - 'nova.scheduler.host_filter.InstanceTypeFilter') - self.assertEquals(hf._full_name(), - 'nova.scheduler.host_filter.InstanceTypeFilter') + hf = host_filter.choose_host_filter('InstanceTypeFilter') + self.assertEquals(hf._full_name().split(".")[-1], 'InstanceTypeFilter') # Test invalid filter ... try: host_filter.choose_host_filter('does not exist') @@ -113,7 +111,7 @@ class HostFilterTestCase(test.TestCase): pass def test_all_host_filter(self): - hf = host_filter.AllHostsFilter() + hf = filters.AllHostsFilter() cooked = hf.instance_type_to_filter(self.instance_type) hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(10, len(hosts)) @@ -121,11 +119,10 @@ class HostFilterTestCase(test.TestCase): self.assertTrue(host.startswith('host')) def test_instance_type_filter(self): - hf = host_filter.InstanceTypeFilter() + hf = filters.InstanceTypeFilter() # filter all hosts that can support 50 ram and 500 disk name, cooked = hf.instance_type_to_filter(self.instance_type) - self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', - name) + self.assertEquals(name.split(".")[-1], 'InstanceTypeFilter') hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -134,21 +131,20 @@ class HostFilterTestCase(test.TestCase): self.assertEquals('host10', just_hosts[5]) def test_instance_type_filter_extra_specs(self): - hf = host_filter.InstanceTypeFilter() + hf = filters.InstanceTypeFilter() # filter all hosts that can support 50 ram and 500 disk name, cooked = hf.instance_type_to_filter(self.gpu_instance_type) - self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', - name) + self.assertEquals(name.split(".")[-1], 'InstanceTypeFilter') hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(1, len(hosts)) just_hosts = [host for host, caps in hosts] self.assertEquals('host07', just_hosts[0]) def test_json_filter(self): - hf = host_filter.JsonFilter() + hf = filters.JsonFilter() # filter all hosts that can support 50 ram and 500 disk name, cooked = hf.instance_type_to_filter(self.instance_type) - self.assertEquals('nova.scheduler.host_filter.JsonFilter', name) + self.assertEquals(name.split(".")[-1], 'JsonFilter') hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -191,6 +187,12 @@ class HostFilterTestCase(test.TestCase): raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) + def debug(*args): + with file("/tmp/debug", "a") as dbg: + msg = " ".join([str(arg) for arg in args]) + dbg.write("%s\n" % msg) + + debug("cooked", cooked, type(cooked)) hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(5, len(hosts))