Refactor scheduling filters

This changes the scheduling filters to use the code in nova/loadables.py
to locate filtering classes.  This also creates some base functionality
in nova/filters.py to be used by both the host scheduler and the cells
scheduler.

The scheduler_available_filters default has changed to
'nova.scheduler.filters.all_filters' which is better named compared to
the old setting of 'standard_filters'.  The old method is still
supported for those that have put it explicitly in their configs, but
it's marked as deprecated.

DocImpact

Change-Id: I84fdeafdba0275ab4b25f8857563bd7b1494bb69

nova/filters.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2011-2012 OpenStack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Filter support
+"""
+
+from nova import loadables
+
+
+class BaseFilter(object):
+    """Base class for all filter classes."""
+    def _filter_one(self, obj, filter_properties):
+        """Return True if it passes the filter, False otherwise.
+        Override this in a subclass.
+        """
+        return True
+
+    def filter_all(self, filter_obj_list, filter_properties):
+        """Yield objects that pass the filter.
+
+        Can be overriden in a subclass, if you need to base filtering
+        decisions on all objects.  Otherwise, one can just override
+        _filter_one() to filter a single object.
+        """
+        for obj in filter_obj_list:
+            if self._filter_one(obj, filter_properties):
+                yield obj
+
+
+class BaseFilterHandler(loadables.BaseLoader):
+    """Base class to handle loading filter classes.
+
+    This class should be subclassed where one needs to use filters.
+    """
+
+    def get_filtered_objects(self, filter_classes, objs,
+            filter_properties):
+        for filter_cls in filter_classes:
+            objs = filter_cls().filter_all(objs, filter_properties)
+        return list(objs)

nova/scheduler/filter_scheduler.py

@@ -268,7 +268,7 @@ class FilterScheduler(driver.Scheduler):
             num_instances = request_spec.get('num_instances', 1)
         for num in xrange(num_instances):
             # Filter local hosts based on requirements ...
-            hosts = self.host_manager.filter_hosts(hosts,
+            hosts = self.host_manager.get_filtered_hosts(hosts,
                     filter_properties)
             if not hosts:
                 # Can't get any more locally.

nova/scheduler/filters/__init__.py

@@ -17,71 +17,41 @@
 Scheduler host filters
 """
 
-import os
+from nova import filters
-import types
+from nova.openstack.common import log as logging
 
-from nova import exception
+LOG = logging.getLogger(__name__)
-from nova.openstack.common import importutils
 
 
-class BaseHostFilter(object):
+class BaseHostFilter(filters.BaseFilter):
     """Base class for host filters."""
+    def _filter_one(self, obj, filter_properties):
+        """Return True if the object passes the filter, otherwise False."""
+        return self.host_passes(obj, filter_properties)
 
     def host_passes(self, host_state, filter_properties):
+        """Return True if the HostState passes the filter, otherwise False.
+        Override this in a subclass.
+        """
         raise NotImplementedError()
 
-    def _full_name(self):
+
-        """module.classname of the filter."""
+class HostFilterHandler(filters.BaseFilterHandler):
-        return "%s.%s" % (self.__module__, self.__class__.__name__)
+    def __init__(self):
+        super(HostFilterHandler, self).__init__(BaseHostFilter)
 
 
-def _is_filter_class(cls):
+def all_filters():
-    """Return whether a class is a valid Host Filter class."""
+    """Return a list of filter classes found in this directory.
-    return type(cls) is types.TypeType and issubclass(cls, BaseHostFilter)
 
-
+    This method is used as the default for available scheduler filters
-def _get_filter_classes_from_module(module_name):
+    and should return a list of all filter classes available.
-    """Get all filter classes from a module."""
+    """
-    classes = []
+    return HostFilterHandler().get_all_classes()
-    module = importutils.import_module(module_name)
-    for obj_name in dir(module):
-        itm = getattr(module, obj_name)
-        if _is_filter_class(itm):
-            classes.append(itm)
-    return classes
 
 
 def standard_filters():
-    """Return a list of filter classes found in this directory."""
+    """Deprecated.  Configs should change to use all_filters()."""
-    classes = []
+    LOG.deprecated(_("Use 'nova.scheduler.filters.all_filters' instead "
-    filters_dir = __path__[0]
+            "of 'nova.scheduler.filters.standard_filters'"))
-    for dirpath, dirnames, filenames in os.walk(filters_dir):
+    return all_filters()
-        relpath = os.path.relpath(dirpath, filters_dir)
-        if relpath == '.':
-            relpkg = ''
-        else:
-            relpkg = '.%s' % '.'.join(relpath.split(os.sep))
-        for fname in filenames:
-            root, ext = os.path.splitext(fname)
-            if ext != '.py' or root == '__init__':
-                continue
-            module_name = "%s%s.%s" % (__package__, relpkg, root)
-            mod_classes = _get_filter_classes_from_module(module_name)
-            classes.extend(mod_classes)
-    return classes
-
-
-def get_filter_classes(filter_class_names):
-    """Get filter classes from class names."""
-    classes = []
-    for cls_name in filter_class_names:
-        obj = importutils.import_class(cls_name)
-        if _is_filter_class(obj):
-            classes.append(obj)
-        elif type(obj) is types.FunctionType:
-            # Get list of classes from a function
-            classes.extend(obj())
-        else:
-            raise exception.ClassNotFound(class_name=cls_name,
-                    exception='Not a valid scheduler filter')
-    return classes

nova/scheduler/host_manager.py

@@ -31,7 +31,7 @@ from nova.scheduler import filters
 
 host_manager_opts = [
     cfg.MultiStrOpt('scheduler_available_filters',
-            default=['nova.scheduler.filters.standard_filters'],
+            default=['nova.scheduler.filters.all_filters'],
             help='Filter classes available to the scheduler which may '
                     'be specified more than once.  An entry of '
                     '"nova.scheduler.filters.standard_filters" '
@@ -239,32 +239,6 @@ class HostState(object):
     def _statmap(self, stats):
         return dict((st['key'], st['value']) for st in stats)
 
-    def passes_filters(self, filter_fns, filter_properties):
-        """Return whether or not this host passes filters."""
-
-        if self.host in filter_properties.get('ignore_hosts', []):
-            LOG.debug(_('Host filter fails for ignored host %(host)s'),
-                      {'host': self.host})
-            return False
-
-        force_hosts = filter_properties.get('force_hosts', [])
-        if force_hosts:
-            if not self.host in force_hosts:
-                LOG.debug(_('Host filter fails for non-forced host %(host)s'),
-                          {'host': self.host})
-            return self.host in force_hosts
-
-        for filter_fn in filter_fns:
-            if not filter_fn(self, filter_properties):
-                LOG.debug(_('Host filter function %(func)s failed for '
-                            '%(host)s'),
-                          {'func': repr(filter_fn),
-                           'host': self.host})
-                return False
-
-        LOG.debug(_('Host filter passes for %(host)s'), {'host': self.host})
-        return True
-
     def __repr__(self):
         return ("(%s, %s) ram:%s disk:%s io_ops:%s instances:%s vm_type:%s" %
                 (self.host, self.nodename, self.free_ram_mb, self.free_disk_mb,
@@ -281,32 +255,28 @@ class HostManager(object):
         # { (host, hypervisor_hostname) : { <service> : { cap k : v }}}
         self.service_states = {}
         self.host_state_map = {}
-        self.filter_classes = filters.get_filter_classes(
+        self.filter_handler = filters.HostFilterHandler()
+        self.filter_classes = self.filter_handler.get_matching_classes(
                 CONF.scheduler_available_filters)
 
-    def _choose_host_filters(self, filters):
+    def _choose_host_filters(self, filter_cls_names):
         """Since the caller may specify which filters to use we need
         to have an authoritative list of what is permissible. This
         function checks the filter names against a predefined set
         of acceptable filters.
         """
-        if filters is None:
+        if filter_cls_names is None:
-            filters = CONF.scheduler_default_filters
+            filter_cls_names = CONF.scheduler_default_filters
-        if not isinstance(filters, (list, tuple)):
+        if not isinstance(filter_cls_names, (list, tuple)):
-            filters = [filters]
+            filter_cls_names = [filter_cls_names]
         good_filters = []
         bad_filters = []
-        for filter_name in filters:
+        for filter_name in filter_cls_names:
             found_class = False
             for cls in self.filter_classes:
                 if cls.__name__ == filter_name:
+                    good_filters.append(cls)
                     found_class = True
-                    filter_instance = cls()
-                    # Get the filter function
-                    filter_func = getattr(filter_instance,
-                            'host_passes', None)
-                    if filter_func:
-                        good_filters.append(filter_func)
                     break
             if not found_class:
                 bad_filters.append(filter_name)
@@ -315,14 +285,36 @@ class HostManager(object):
             raise exception.SchedulerHostFilterNotFound(filter_name=msg)
         return good_filters
 
-    def filter_hosts(self, hosts, filter_properties, filters=None):
+    def get_filtered_hosts(self, hosts, filter_properties,
+            filter_class_names=None):
         """Filter hosts and return only ones passing all filters"""
-        filtered_hosts = []
+        filter_classes = self._choose_host_filters(filter_class_names)
-        filter_fns = self._choose_host_filters(filters)
+
-        for host in hosts:
+        hosts = set(hosts)
-            if host.passes_filters(filter_fns, filter_properties):
+        ignore_hosts = set(filter_properties.get('ignore_hosts', []))
-                filtered_hosts.append(host)
+        ignore_hosts = hosts & ignore_hosts
-        return filtered_hosts
+        if ignore_hosts:
+            ignored_hosts = ', '.join(ignore_hosts)
+            msg = _('Host filter ignoring hosts: %(ignored_hosts)s')
+            LOG.debug(msg, locals())
+            hosts = hosts - ignore_hosts
+
+        force_hosts = set(filter_properties.get('force_hosts', []))
+        if force_hosts:
+            matching_force_hosts = hosts & force_hosts
+            if not matching_force_hosts:
+                forced_hosts = ', '.join(force_hosts)
+                msg = _("No hosts matched due to not matching 'force_hosts'"
+                        "value of '%(forced_hosts)s'")
+                LOG.debug(msg, locals())
+                return []
+            forced_hosts = ', '.join(matching_force_hosts)
+            msg = _('Host filter forcing available hosts to %(forced_hosts)s')
+            LOG.debug(msg, locals())
+            hosts = matching_force_hosts
+
+        return self.filter_handler.get_filtered_objects(filter_classes,
+                hosts, filter_properties)
 
     def update_service_capabilities(self, service_name, host, capabilities):
         """Update the per-service capabilities based on this notification."""

nova/tests/scheduler/test_filter_scheduler.py

@@ -32,7 +32,7 @@ from nova.tests.scheduler import fakes
 from nova.tests.scheduler import test_scheduler
 
 
-def fake_filter_hosts(hosts, filter_properties):
+def fake_get_filtered_hosts(hosts, filter_properties):
     return list(hosts)
 
 
@@ -155,8 +155,8 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
         fake_context = context.RequestContext('user', 'project',
                 is_admin=True)
 
-        self.stubs.Set(sched.host_manager, 'filter_hosts',
+        self.stubs.Set(sched.host_manager, 'get_filtered_hosts',
-                fake_filter_hosts)
+                fake_get_filtered_hosts)
         self.stubs.Set(least_cost, 'weighted_sum', _fake_weighted_sum)
         fakes.mox_host_manager_db_calls(self.mox, fake_context)
 

nova/tests/scheduler/test_host_filters.py

@@ -263,32 +263,33 @@ class HostFiltersTestCase(test.TestCase):
         self.json_query = jsonutils.dumps(
                 ['and', ['>=', '$free_ram_mb', 1024],
                         ['>=', '$free_disk_mb', 200 * 1024]])
-        # This has a side effect of testing 'get_filter_classes'
+        filter_handler = filters.HostFilterHandler()
-        # when specifying a method (in this case, our standard filters)
+        classes = filter_handler.get_matching_classes(
-        classes = filters.get_filter_classes(
+                ['nova.scheduler.filters.all_filters'])
-                ['nova.scheduler.filters.standard_filters'])
         self.class_map = {}
         for cls in classes:
             self.class_map[cls.__name__] = cls
 
-    def test_get_filter_classes(self):
+    def test_standard_filters_is_deprecated(self):
-        classes = filters.get_filter_classes(
+        info = {'called': False}
-                ['nova.tests.scheduler.test_host_filters.TestFilter'])
-        self.assertEqual(len(classes), 1)
-        self.assertEqual(classes[0].__name__, 'TestFilter')
-        # Test a specific class along with our standard filters
-        classes = filters.get_filter_classes(
-                ['nova.tests.scheduler.test_host_filters.TestFilter',
-                 'nova.scheduler.filters.standard_filters'])
-        self.assertEqual(len(classes), 1 + len(self.class_map))
 
-    def test_get_filter_classes_raises_on_invalid_classes(self):
+        def _fake_deprecated(*args, **kwargs):
-        self.assertRaises(ImportError,
+            info['called'] = True
-                filters.get_filter_classes,
+
-                ['nova.tests.scheduler.test_host_filters.NoExist'])
+        self.stubs.Set(filters.LOG, 'deprecated', _fake_deprecated)
-        self.assertRaises(exception.ClassNotFound,
+
-                filters.get_filter_classes,
+        filter_handler = filters.HostFilterHandler()
-                ['nova.tests.scheduler.test_host_filters.TestBogusFilter'])
+        filter_handler.get_matching_classes(
+                ['nova.scheduler.filters.standard_filters'])
+
+        self.assertTrue(info['called'])
+        self.assertIn('AllHostsFilter', self.class_map)
+        self.assertIn('ComputeFilter', self.class_map)
+
+    def test_all_filters(self):
+        # Double check at least a couple of known filters exist
+        self.assertIn('AllHostsFilter', self.class_map)
+        self.assertIn('ComputeFilter', self.class_map)
 
     def test_all_host_filter(self):
         filt_cls = self.class_map['AllHostsFilter']()

nova/tests/scheduler/test_host_manager.py

@@ -62,34 +62,146 @@ class HostManagerTestCase(test.TestCase):
                 ComputeFilterClass2]
 
         # Test we returns 1 correct function
-        filter_fns = self.host_manager._choose_host_filters(None)
+        filter_classes = self.host_manager._choose_host_filters(None)
-        self.assertEqual(len(filter_fns), 1)
+        self.assertEqual(len(filter_classes), 1)
-        self.assertEqual(filter_fns[0].__func__,
+        self.assertEqual(filter_classes[0].__name__, 'ComputeFilterClass2')
-                ComputeFilterClass2.host_passes.__func__)
 
-    def test_filter_hosts(self):
+    def test_get_filtered_hosts(self):
-        filters = ['fake-filter1', 'fake-filter2']
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
-        fake_host1 = host_manager.HostState('host1', 'node1')
+                'fake_host1']
-        fake_host2 = host_manager.HostState('host2', 'node2')
+        fake_classes = 'fake_classes'
-        hosts = [fake_host1, fake_host2]
+        fake_properties = {'moo': 1, 'cow': 2}
-        filter_properties = {'fake_prop': 'fake_val'}
+        expected_hosts = set(fake_hosts)
+        fake_result = 'fake_result'
 
-        self.mox.StubOutWithMock(self.host_manager,
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
-                '_choose_host_filters')
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
-        self.mox.StubOutWithMock(fake_host1, 'passes_filters')
+                'get_filtered_objects')
-        self.mox.StubOutWithMock(fake_host2, 'passes_filters')
 
-        self.host_manager._choose_host_filters(None).AndReturn(filters)
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
-        fake_host1.passes_filters(filters, filter_properties).AndReturn(
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
-                False)
+                expected_hosts, fake_properties).AndReturn(fake_result)
-        fake_host2.passes_filters(filters, filter_properties).AndReturn(
-                True)
 
         self.mox.ReplayAll()
-        filtered_hosts = self.host_manager.filter_hosts(hosts,
+
-                filter_properties, filters=None)
+        result = self.host_manager. get_filtered_hosts(fake_hosts,
-        self.assertEqual(len(filtered_hosts), 1)
+                fake_properties)
-        self.assertEqual(filtered_hosts[0], fake_host2)
+        self.assertEqual(result, fake_result)
+
+    def test_get_filtered_hosts_with_specificed_filters(self):
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1']
+        fake_classes = 'fake_classes'
+        fake_properties = {'moo': 1, 'cow': 2}
+        fake_filters = 'fake_filters'
+        expected_hosts = set(fake_hosts)
+        fake_result = 'fake_result'
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+
+        self.host_manager._choose_host_filters(fake_filters).AndReturn(
+                fake_classes)
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
+                expected_hosts, fake_properties).AndReturn(fake_result)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties, filter_class_names=fake_filters)
+        self.assertEqual(result, fake_result)
+
+    def test_get_filtered_hosts_with_ignore(self):
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1', 'fake_host3', 'fake_host4']
+        fake_classes = 'fake_classes'
+        fake_properties = {'ignore_hosts': ['fake_host1', 'fake_host3',
+            'fake_host5']}
+        expected_hosts = set(['fake_host2', 'fake_host4'])
+        fake_result = 'fake_result'
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
+                expected_hosts, fake_properties).AndReturn(fake_result)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties)
+        self.assertEqual(result, fake_result)
+
+    def test_get_filtered_hosts_with_force_hosts(self):
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1', 'fake_host3', 'fake_host4']
+        fake_classes = 'fake_classes'
+        fake_properties = {'force_hosts': ['fake_host1', 'fake_host3',
+            'fake_host5']}
+        expected_hosts = set(['fake_host1', 'fake_host3'])
+        fake_result = 'fake_result'
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
+                expected_hosts, fake_properties).AndReturn(fake_result)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties)
+        self.assertEqual(result, fake_result)
+
+    def test_get_filtered_hosts_with_no_matching_force_hosts(self):
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1', 'fake_host3', 'fake_host4']
+        fake_classes = 'fake_classes'
+        fake_properties = {'force_hosts': ['fake_host5', 'fake_host6']}
+        expected_result = []
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        # Make sure this is not called.
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties)
+        self.assertEqual(result, expected_result)
+
+    def test_get_filtered_hosts_with_ignore_and_force(self):
+        """Ensure ignore_hosts processed before force_hosts in host filters"""
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1', 'fake_host3', 'fake_host4']
+        fake_classes = 'fake_classes'
+        fake_properties = {'force_hosts': ['fake_host3', 'fake_host1'],
+                           'ignore_hosts': ['fake_host1']}
+        expected_hosts = set(['fake_host3'])
+        fake_result = 'fake_result'
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        # Make sure this is not called.
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
+                expected_hosts, fake_properties).AndReturn(fake_result)
+
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties)
+        self.assertEqual(result, fake_result)
 
     def test_update_service_capabilities(self):
         service_states = self.host_manager.service_states
@@ -190,91 +302,6 @@ class HostStateTestCase(test.TestCase):
     # update_from_compute_node() and consume_from_instance() are tested
     # in HostManagerTestCase.test_get_all_host_states()
 
-    def test_host_state_passes_filters_passes(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        cls1.host_passes(fake_host, filter_properties).AndReturn(True)
-        cls2.host_passes(fake_host, filter_properties).AndReturn(True)
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertTrue(result)
-
-    def test_host_state_passes_filters_passes_with_ignore(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {'ignore_hosts': ['host2']}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        cls1.host_passes(fake_host, filter_properties).AndReturn(True)
-        cls2.host_passes(fake_host, filter_properties).AndReturn(True)
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertTrue(result)
-
-    def test_host_state_passes_filters_fails(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        cls1.host_passes(fake_host, filter_properties).AndReturn(False)
-        # cls2.host_passes() not called because of short circuit
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertFalse(result)
-
-    def test_host_state_passes_filters_fails_from_ignore(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {'ignore_hosts': ['host1']}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        # cls[12].host_passes() not called because of short circuit
-        # with matching host to ignore
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertFalse(result)
-
-    def test_host_state_passes_filters_skipped_from_force(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {'force_hosts': ['host1']}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        # cls[12].host_passes() not called because of short circuit
-        # with matching host to force
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertTrue(result)
-
     def test_stat_consumption_from_compute_node(self):
         stats = [
             dict(key='num_instances', value='5'),

nova/tests/test_filters.py

@@ -0,0 +1,125 @@
+# Copyright 2012 OpenStack LLC.  # All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests For Scheduler Host Filters.
+"""
+
+import inspect
+import sys
+
+from nova import filters
+from nova import loadables
+from nova import test
+
+
+class Filter1(filters.BaseFilter):
+    """Test Filter class #1."""
+    pass
+
+
+class Filter2(filters.BaseFilter):
+    """Test Filter class #2."""
+    pass
+
+
+class FiltersTestCase(test.TestCase):
+    def test_filter_all(self):
+        filter_obj_list = ['obj1', 'obj2', 'obj3']
+        filter_properties = 'fake_filter_properties'
+        base_filter = filters.BaseFilter()
+
+        self.mox.StubOutWithMock(base_filter, '_filter_one')
+
+        base_filter._filter_one('obj1', filter_properties).AndReturn(True)
+        base_filter._filter_one('obj2', filter_properties).AndReturn(False)
+        base_filter._filter_one('obj3', filter_properties).AndReturn(True)
+
+        self.mox.ReplayAll()
+
+        result = base_filter.filter_all(filter_obj_list, filter_properties)
+        self.assertTrue(inspect.isgenerator(result))
+        self.assertEqual(list(result), ['obj1', 'obj3'])
+
+    def test_filter_all_recursive_yields(self):
+        """Test filter_all() allows generators from previous filter_all()s."""
+        # filter_all() yields results.  We want to make sure that we can
+        # call filter_all() with generators returned from previous calls
+        # to filter_all().
+        filter_obj_list = ['obj1', 'obj2', 'obj3']
+        filter_properties = 'fake_filter_properties'
+        base_filter = filters.BaseFilter()
+
+        self.mox.StubOutWithMock(base_filter, '_filter_one')
+
+        total_iterations = 200
+
+        # The order that _filter_one is going to get called gets
+        # confusing because we will be recursively yielding things..
+        # We are going to simulate the first call to filter_all()
+        # returning False for 'obj2'.  So, 'obj1' will get yielded
+        # 'total_iterations' number of times before the first filter_all()
+        # call gets to processing 'obj2'.  We then return 'False' for it.
+        # After that, 'obj3' gets yielded 'total_iterations' number of
+        # times.
+        for x in xrange(total_iterations):
+            base_filter._filter_one('obj1', filter_properties).AndReturn(True)
+        base_filter._filter_one('obj2', filter_properties).AndReturn(False)
+        for x in xrange(total_iterations):
+            base_filter._filter_one('obj3', filter_properties).AndReturn(True)
+        self.mox.ReplayAll()
+
+        objs = iter(filter_obj_list)
+        for x in xrange(total_iterations):
+            # Pass in generators returned from previous calls.
+            objs = base_filter.filter_all(objs, filter_properties)
+        self.assertTrue(inspect.isgenerator(objs))
+        self.assertEqual(list(objs), ['obj1', 'obj3'])
+
+    def test_get_filtered_objects(self):
+        filter_objs_initial = ['initial', 'filter1', 'objects1']
+        filter_objs_second = ['second', 'filter2', 'objects2']
+        filter_objs_last = ['last', 'filter3', 'objects3']
+        filter_properties = 'fake_filter_properties'
+
+        def _fake_base_loader_init(*args, **kwargs):
+            pass
+
+        self.stubs.Set(loadables.BaseLoader, '__init__',
+                       _fake_base_loader_init)
+
+        filt1_mock = self.mox.CreateMock(Filter1)
+        filt2_mock = self.mox.CreateMock(Filter2)
+
+        self.mox.StubOutWithMock(sys.modules[__name__], 'Filter1',
+                                 use_mock_anything=True)
+        self.mox.StubOutWithMock(filt1_mock, 'filter_all')
+        self.mox.StubOutWithMock(sys.modules[__name__], 'Filter2',
+                                 use_mock_anything=True)
+        self.mox.StubOutWithMock(filt2_mock, 'filter_all')
+
+        Filter1().AndReturn(filt1_mock)
+        filt1_mock.filter_all(filter_objs_initial,
+                              filter_properties).AndReturn(filter_objs_second)
+        Filter2().AndReturn(filt2_mock)
+        filt2_mock.filter_all(filter_objs_second,
+                              filter_properties).AndReturn(filter_objs_last)
+
+        self.mox.ReplayAll()
+
+        filter_handler = filters.BaseFilterHandler(filters.BaseFilter)
+        filter_classes = [Filter1, Filter2]
+        result = filter_handler.get_filtered_objects(filter_classes,
+                                                     filter_objs_initial,
+                                                     filter_properties)
+        self.assertEqual(result, filter_objs_last)