x/gantt
Code Issues Proposed changes

Refactor scheduling filters

Browse Source 
This changes the scheduling filters to use the code in nova/loadables.py
to locate filtering classes.  This also creates some base functionality
in nova/filters.py to be used by both the host scheduler and the cells
scheduler.

The scheduler_available_filters default has changed to
'nova.scheduler.filters.all_filters' which is better named compared to
the old setting of 'standard_filters'.  The old method is still
supported for those that have put it explicitly in their configs, but
it's marked as deprecated.

DocImpact

Change-Id: I84fdeafdba0275ab4b25f8857563bd7b1494bb69
This commit is contained in:
Chris Behrens 2012-11-09 01:57:42 +00:00
parent 8ee7798af7
commit 770acb12ab
8 changed files with 402 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
nova/filters.py Normal file
View File

@ -0,0 +1,53 @@
# Copyright (c) 2011-2012 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Filter support
"""
from nova import loadables
class BaseFilter(object):
"""Base class for all filter classes."""
def _filter_one(self, obj, filter_properties):
"""Return True if it passes the filter, False otherwise.
Override this in a subclass.
"""
return True
def filter_all(self, filter_obj_list, filter_properties):
"""Yield objects that pass the filter.
Can be overriden in a subclass, if you need to base filtering
decisions on all objects. Otherwise, one can just override
_filter_one() to filter a single object.
"""
for obj in filter_obj_list:
if self._filter_one(obj, filter_properties):
yield obj
class BaseFilterHandler(loadables.BaseLoader):
"""Base class to handle loading filter classes.
This class should be subclassed where one needs to use filters.
"""
def get_filtered_objects(self, filter_classes, objs,
filter_properties):
for filter_cls in filter_classes:
objs = filter_cls().filter_all(objs, filter_properties)
return list(objs)

2
nova/scheduler/filter_scheduler.py
View File

@ -268,7 +268,7 @@ class FilterScheduler(driver.Scheduler):
num_instances = request_spec.get('num_instances', 1)
for num in xrange(num_instances):
# Filter local hosts based on requirements ...
hosts = self.host_manager.filter_hosts(hosts,
hosts = self.host_manager.get_filtered_hosts(hosts,
filter_properties)
if not hosts:
# Can't get any more locally.

78
nova/scheduler/filters/__init__.py
View File

@ -17,71 +17,41 @@
Scheduler host filters
"""
import os
import types
from nova import filters
from nova.openstack.common import log as logging
from nova import exception
from nova.openstack.common import importutils
LOG = logging.getLogger(__name__)
class BaseHostFilter(object):
class BaseHostFilter(filters.BaseFilter):
"""Base class for host filters."""
def _filter_one(self, obj, filter_properties):
"""Return True if the object passes the filter, otherwise False."""
return self.host_passes(obj, filter_properties)
def host_passes(self, host_state, filter_properties):
"""Return True if the HostState passes the filter, otherwise False.
Override this in a subclass.
"""
raise NotImplementedError()
def _full_name(self):
"""module.classname of the filter."""
return "%s.%s" % (self.__module__, self.__class__.__name__)
class HostFilterHandler(filters.BaseFilterHandler):
def __init__(self):
super(HostFilterHandler, self).__init__(BaseHostFilter)
def _is_filter_class(cls):
"""Return whether a class is a valid Host Filter class."""
return type(cls) is types.TypeType and issubclass(cls, BaseHostFilter)
def all_filters():
"""Return a list of filter classes found in this directory.
def _get_filter_classes_from_module(module_name):
"""Get all filter classes from a module."""
classes = []
module = importutils.import_module(module_name)
for obj_name in dir(module):
itm = getattr(module, obj_name)
if _is_filter_class(itm):
classes.append(itm)
return classes
This method is used as the default for available scheduler filters
and should return a list of all filter classes available.
"""
return HostFilterHandler().get_all_classes()
def standard_filters():
"""Return a list of filter classes found in this directory."""
classes = []
filters_dir = __path__[0]
for dirpath, dirnames, filenames in os.walk(filters_dir):
relpath = os.path.relpath(dirpath, filters_dir)
if relpath == '.':
relpkg = ''
else:
relpkg = '.%s' % '.'.join(relpath.split(os.sep))
for fname in filenames:
root, ext = os.path.splitext(fname)
if ext != '.py' or root == '__init__':
continue
module_name = "%s%s.%s" % (__package__, relpkg, root)
mod_classes = _get_filter_classes_from_module(module_name)
classes.extend(mod_classes)
return classes
def get_filter_classes(filter_class_names):
"""Get filter classes from class names."""
classes = []
for cls_name in filter_class_names:
obj = importutils.import_class(cls_name)
if _is_filter_class(obj):
classes.append(obj)
elif type(obj) is types.FunctionType:
# Get list of classes from a function
classes.extend(obj())
else:
raise exception.ClassNotFound(class_name=cls_name,
exception='Not a valid scheduler filter')
return classes
"""Deprecated. Configs should change to use all_filters()."""
LOG.deprecated(_("Use 'nova.scheduler.filters.all_filters' instead "
"of 'nova.scheduler.filters.standard_filters'"))
return all_filters()

86
nova/scheduler/host_manager.py
View File

@ -31,7 +31,7 @@ from nova.scheduler import filters
host_manager_opts = [
cfg.MultiStrOpt('scheduler_available_filters',
default=['nova.scheduler.filters.standard_filters'],
default=['nova.scheduler.filters.all_filters'],
help='Filter classes available to the scheduler which may '
'be specified more than once. An entry of '
'"nova.scheduler.filters.standard_filters" '
@ -239,32 +239,6 @@ class HostState(object):
def _statmap(self, stats):
return dict((st['key'], st['value']) for st in stats)
def passes_filters(self, filter_fns, filter_properties):
"""Return whether or not this host passes filters."""
if self.host in filter_properties.get('ignore_hosts', []):
LOG.debug(_('Host filter fails for ignored host %(host)s'),
{'host': self.host})
return False
force_hosts = filter_properties.get('force_hosts', [])
if force_hosts:
if not self.host in force_hosts:
LOG.debug(_('Host filter fails for non-forced host %(host)s'),
{'host': self.host})
return self.host in force_hosts
for filter_fn in filter_fns:
if not filter_fn(self, filter_properties):
LOG.debug(_('Host filter function %(func)s failed for '
'%(host)s'),
{'func': repr(filter_fn),
'host': self.host})
return False
LOG.debug(_('Host filter passes for %(host)s'), {'host': self.host})
return True
def __repr__(self):
return ("(%s, %s) ram:%s disk:%s io_ops:%s instances:%s vm_type:%s" %
(self.host, self.nodename, self.free_ram_mb, self.free_disk_mb,
@ -281,32 +255,28 @@ class HostManager(object):
# { (host, hypervisor_hostname) : { <service> : { cap k : v }}}
self.service_states = {}
self.host_state_map = {}
self.filter_classes = filters.get_filter_classes(
self.filter_handler = filters.HostFilterHandler()
self.filter_classes = self.filter_handler.get_matching_classes(
CONF.scheduler_available_filters)
def _choose_host_filters(self, filters):
def _choose_host_filters(self, filter_cls_names):
"""Since the caller may specify which filters to use we need
to have an authoritative list of what is permissible. This
function checks the filter names against a predefined set
of acceptable filters.
"""
if filters is None:
filters = CONF.scheduler_default_filters
if not isinstance(filters, (list, tuple)):
filters = [filters]
if filter_cls_names is None:
filter_cls_names = CONF.scheduler_default_filters
if not isinstance(filter_cls_names, (list, tuple)):
filter_cls_names = [filter_cls_names]
good_filters = []
bad_filters = []
for filter_name in filters:
for filter_name in filter_cls_names:
found_class = False
for cls in self.filter_classes:
if cls.__name__ == filter_name:
good_filters.append(cls)
found_class = True
filter_instance = cls()
# Get the filter function
filter_func = getattr(filter_instance,
'host_passes', None)
if filter_func:
good_filters.append(filter_func)
break
if not found_class:
bad_filters.append(filter_name)
@ -315,14 +285,36 @@ class HostManager(object):
raise exception.SchedulerHostFilterNotFound(filter_name=msg)
return good_filters
def filter_hosts(self, hosts, filter_properties, filters=None):
def get_filtered_hosts(self, hosts, filter_properties,
filter_class_names=None):
"""Filter hosts and return only ones passing all filters"""
filtered_hosts = []
filter_fns = self._choose_host_filters(filters)
for host in hosts:
if host.passes_filters(filter_fns, filter_properties):
filtered_hosts.append(host)
return filtered_hosts
filter_classes = self._choose_host_filters(filter_class_names)
hosts = set(hosts)
ignore_hosts = set(filter_properties.get('ignore_hosts', []))
ignore_hosts = hosts & ignore_hosts
if ignore_hosts:
ignored_hosts = ', '.join(ignore_hosts)
msg = _('Host filter ignoring hosts: %(ignored_hosts)s')
LOG.debug(msg, locals())
hosts = hosts - ignore_hosts
force_hosts = set(filter_properties.get('force_hosts', []))
if force_hosts:
matching_force_hosts = hosts & force_hosts
if not matching_force_hosts:
forced_hosts = ', '.join(force_hosts)
msg = _("No hosts matched due to not matching 'force_hosts'"
"value of '%(forced_hosts)s'")
LOG.debug(msg, locals())
return []
forced_hosts = ', '.join(matching_force_hosts)
msg = _('Host filter forcing available hosts to %(forced_hosts)s')
LOG.debug(msg, locals())
hosts = matching_force_hosts
return self.filter_handler.get_filtered_objects(filter_classes,
hosts, filter_properties)
def update_service_capabilities(self, service_name, host, capabilities):
"""Update the per-service capabilities based on this notification."""

6
nova/tests/scheduler/test_filter_scheduler.py
View File

@ -32,7 +32,7 @@ from nova.tests.scheduler import fakes
from nova.tests.scheduler import test_scheduler
def fake_filter_hosts(hosts, filter_properties):
def fake_get_filtered_hosts(hosts, filter_properties):
return list(hosts)
@ -155,8 +155,8 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
fake_context = context.RequestContext('user', 'project',
is_admin=True)
self.stubs.Set(sched.host_manager, 'filter_hosts',
fake_filter_hosts)
self.stubs.Set(sched.host_manager, 'get_filtered_hosts',
fake_get_filtered_hosts)
self.stubs.Set(least_cost, 'weighted_sum', _fake_weighted_sum)
fakes.mox_host_manager_db_calls(self.mox, fake_context)

43
nova/tests/scheduler/test_host_filters.py
View File

@ -263,32 +263,33 @@ class HostFiltersTestCase(test.TestCase):
self.json_query = jsonutils.dumps(
['and', ['>=', '$free_ram_mb', 1024],
['>=', '$free_disk_mb', 200 * 1024]])
# This has a side effect of testing 'get_filter_classes'
# when specifying a method (in this case, our standard filters)
classes = filters.get_filter_classes(
['nova.scheduler.filters.standard_filters'])
filter_handler = filters.HostFilterHandler()
classes = filter_handler.get_matching_classes(
['nova.scheduler.filters.all_filters'])
self.class_map = {}
for cls in classes:
self.class_map[cls.__name__] = cls
def test_get_filter_classes(self):
classes = filters.get_filter_classes(
['nova.tests.scheduler.test_host_filters.TestFilter'])
self.assertEqual(len(classes), 1)
self.assertEqual(classes[0].__name__, 'TestFilter')
# Test a specific class along with our standard filters
classes = filters.get_filter_classes(
['nova.tests.scheduler.test_host_filters.TestFilter',
'nova.scheduler.filters.standard_filters'])
self.assertEqual(len(classes), 1 + len(self.class_map))
def test_standard_filters_is_deprecated(self):
info = {'called': False}
def test_get_filter_classes_raises_on_invalid_classes(self):
self.assertRaises(ImportError,
filters.get_filter_classes,
['nova.tests.scheduler.test_host_filters.NoExist'])
self.assertRaises(exception.ClassNotFound,
filters.get_filter_classes,
['nova.tests.scheduler.test_host_filters.TestBogusFilter'])
def _fake_deprecated(*args, **kwargs):
info['called'] = True
self.stubs.Set(filters.LOG, 'deprecated', _fake_deprecated)
filter_handler = filters.HostFilterHandler()
filter_handler.get_matching_classes(
['nova.scheduler.filters.standard_filters'])
self.assertTrue(info['called'])
self.assertIn('AllHostsFilter', self.class_map)
self.assertIn('ComputeFilter', self.class_map)
def test_all_filters(self):
# Double check at least a couple of known filters exist
self.assertIn('AllHostsFilter', self.class_map)
self.assertIn('ComputeFilter', self.class_map)
def test_all_host_filter(self):
filt_cls = self.class_map['AllHostsFilter']()

243
nova/tests/scheduler/test_host_manager.py
View File

@ -62,34 +62,146 @@ class HostManagerTestCase(test.TestCase):
ComputeFilterClass2]
# Test we returns 1 correct function
filter_fns = self.host_manager._choose_host_filters(None)
self.assertEqual(len(filter_fns), 1)
self.assertEqual(filter_fns[0].__func__,
ComputeFilterClass2.host_passes.__func__)
filter_classes = self.host_manager._choose_host_filters(None)
self.assertEqual(len(filter_classes), 1)
self.assertEqual(filter_classes[0].__name__, 'ComputeFilterClass2')
def test_filter_hosts(self):
filters = ['fake-filter1', 'fake-filter2']
fake_host1 = host_manager.HostState('host1', 'node1')
fake_host2 = host_manager.HostState('host2', 'node2')
hosts = [fake_host1, fake_host2]
filter_properties = {'fake_prop': 'fake_val'}
def test_get_filtered_hosts(self):
fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
'fake_host1']
fake_classes = 'fake_classes'
fake_properties = {'moo': 1, 'cow': 2}
expected_hosts = set(fake_hosts)
fake_result = 'fake_result'
self.mox.StubOutWithMock(self.host_manager,
'_choose_host_filters')
self.mox.StubOutWithMock(fake_host1, 'passes_filters')
self.mox.StubOutWithMock(fake_host2, 'passes_filters')
self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
self.mox.StubOutWithMock(self.host_manager.filter_handler,
'get_filtered_objects')
self.host_manager._choose_host_filters(None).AndReturn(filters)
fake_host1.passes_filters(filters, filter_properties).AndReturn(
False)
fake_host2.passes_filters(filters, filter_properties).AndReturn(
True)
self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
self.host_manager.filter_handler.get_filtered_objects(fake_classes,
expected_hosts, fake_properties).AndReturn(fake_result)
self.mox.ReplayAll()
filtered_hosts = self.host_manager.filter_hosts(hosts,
filter_properties, filters=None)
self.assertEqual(len(filtered_hosts), 1)
self.assertEqual(filtered_hosts[0], fake_host2)
result = self.host_manager. get_filtered_hosts(fake_hosts,
fake_properties)
self.assertEqual(result, fake_result)
def test_get_filtered_hosts_with_specificed_filters(self):
fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
'fake_host1']
fake_classes = 'fake_classes'
fake_properties = {'moo': 1, 'cow': 2}
fake_filters = 'fake_filters'
expected_hosts = set(fake_hosts)
fake_result = 'fake_result'
self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
self.mox.StubOutWithMock(self.host_manager.filter_handler,
'get_filtered_objects')
self.host_manager._choose_host_filters(fake_filters).AndReturn(
fake_classes)
self.host_manager.filter_handler.get_filtered_objects(fake_classes,
expected_hosts, fake_properties).AndReturn(fake_result)
self.mox.ReplayAll()
result = self.host_manager.get_filtered_hosts(fake_hosts,
fake_properties, filter_class_names=fake_filters)
self.assertEqual(result, fake_result)
def test_get_filtered_hosts_with_ignore(self):
fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
'fake_host1', 'fake_host3', 'fake_host4']
fake_classes = 'fake_classes'
fake_properties = {'ignore_hosts': ['fake_host1', 'fake_host3',
'fake_host5']}
expected_hosts = set(['fake_host2', 'fake_host4'])
fake_result = 'fake_result'
self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
self.mox.StubOutWithMock(self.host_manager.filter_handler,
'get_filtered_objects')
self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
self.host_manager.filter_handler.get_filtered_objects(fake_classes,
expected_hosts, fake_properties).AndReturn(fake_result)
self.mox.ReplayAll()
result = self.host_manager.get_filtered_hosts(fake_hosts,
fake_properties)
self.assertEqual(result, fake_result)
def test_get_filtered_hosts_with_force_hosts(self):
fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
'fake_host1', 'fake_host3', 'fake_host4']
fake_classes = 'fake_classes'
fake_properties = {'force_hosts': ['fake_host1', 'fake_host3',
'fake_host5']}
expected_hosts = set(['fake_host1', 'fake_host3'])
fake_result = 'fake_result'
self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
self.mox.StubOutWithMock(self.host_manager.filter_handler,
'get_filtered_objects')
self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
self.host_manager.filter_handler.get_filtered_objects(fake_classes,
expected_hosts, fake_properties).AndReturn(fake_result)
self.mox.ReplayAll()
result = self.host_manager.get_filtered_hosts(fake_hosts,
fake_properties)
self.assertEqual(result, fake_result)
def test_get_filtered_hosts_with_no_matching_force_hosts(self):
fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
'fake_host1', 'fake_host3', 'fake_host4']
fake_classes = 'fake_classes'
fake_properties = {'force_hosts': ['fake_host5', 'fake_host6']}
expected_result = []
self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
# Make sure this is not called.
self.mox.StubOutWithMock(self.host_manager.filter_handler,
'get_filtered_objects')
self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
self.mox.ReplayAll()
result = self.host_manager.get_filtered_hosts(fake_hosts,
fake_properties)
self.assertEqual(result, expected_result)
def test_get_filtered_hosts_with_ignore_and_force(self):
"""Ensure ignore_hosts processed before force_hosts in host filters"""
fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
'fake_host1', 'fake_host3', 'fake_host4']
fake_classes = 'fake_classes'
fake_properties = {'force_hosts': ['fake_host3', 'fake_host1'],
'ignore_hosts': ['fake_host1']}
expected_hosts = set(['fake_host3'])
fake_result = 'fake_result'
self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
# Make sure this is not called.
self.mox.StubOutWithMock(self.host_manager.filter_handler,
'get_filtered_objects')
self.host_manager.filter_handler.get_filtered_objects(fake_classes,
expected_hosts, fake_properties).AndReturn(fake_result)
self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
self.mox.ReplayAll()
result = self.host_manager.get_filtered_hosts(fake_hosts,
fake_properties)
self.assertEqual(result, fake_result)
def test_update_service_capabilities(self):
service_states = self.host_manager.service_states
@ -190,91 +302,6 @@ class HostStateTestCase(test.TestCase):
# update_from_compute_node() and consume_from_instance() are tested
# in HostManagerTestCase.test_get_all_host_states()
def test_host_state_passes_filters_passes(self):
fake_host = host_manager.HostState('host1', 'node1')
filter_properties = {}
cls1 = ComputeFilterClass1()
cls2 = ComputeFilterClass2()
self.mox.StubOutWithMock(cls1, 'host_passes')
self.mox.StubOutWithMock(cls2, 'host_passes')
filter_fns = [cls1.host_passes, cls2.host_passes]
cls1.host_passes(fake_host, filter_properties).AndReturn(True)
cls2.host_passes(fake_host, filter_properties).AndReturn(True)
self.mox.ReplayAll()
result = fake_host.passes_filters(filter_fns, filter_properties)
self.assertTrue(result)
def test_host_state_passes_filters_passes_with_ignore(self):
fake_host = host_manager.HostState('host1', 'node1')
filter_properties = {'ignore_hosts': ['host2']}
cls1 = ComputeFilterClass1()
cls2 = ComputeFilterClass2()
self.mox.StubOutWithMock(cls1, 'host_passes')
self.mox.StubOutWithMock(cls2, 'host_passes')
filter_fns = [cls1.host_passes, cls2.host_passes]
cls1.host_passes(fake_host, filter_properties).AndReturn(True)
cls2.host_passes(fake_host, filter_properties).AndReturn(True)
self.mox.ReplayAll()
result = fake_host.passes_filters(filter_fns, filter_properties)
self.assertTrue(result)
def test_host_state_passes_filters_fails(self):
fake_host = host_manager.HostState('host1', 'node1')
filter_properties = {}
cls1 = ComputeFilterClass1()
cls2 = ComputeFilterClass2()
self.mox.StubOutWithMock(cls1, 'host_passes')
self.mox.StubOutWithMock(cls2, 'host_passes')
filter_fns = [cls1.host_passes, cls2.host_passes]
cls1.host_passes(fake_host, filter_properties).AndReturn(False)
# cls2.host_passes() not called because of short circuit
self.mox.ReplayAll()
result = fake_host.passes_filters(filter_fns, filter_properties)
self.assertFalse(result)
def test_host_state_passes_filters_fails_from_ignore(self):
fake_host = host_manager.HostState('host1', 'node1')
filter_properties = {'ignore_hosts': ['host1']}
cls1 = ComputeFilterClass1()
cls2 = ComputeFilterClass2()
self.mox.StubOutWithMock(cls1, 'host_passes')
self.mox.StubOutWithMock(cls2, 'host_passes')
filter_fns = [cls1.host_passes, cls2.host_passes]
# cls[12].host_passes() not called because of short circuit
# with matching host to ignore
self.mox.ReplayAll()
result = fake_host.passes_filters(filter_fns, filter_properties)
self.assertFalse(result)
def test_host_state_passes_filters_skipped_from_force(self):
fake_host = host_manager.HostState('host1', 'node1')
filter_properties = {'force_hosts': ['host1']}
cls1 = ComputeFilterClass1()
cls2 = ComputeFilterClass2()
self.mox.StubOutWithMock(cls1, 'host_passes')
self.mox.StubOutWithMock(cls2, 'host_passes')
filter_fns = [cls1.host_passes, cls2.host_passes]
# cls[12].host_passes() not called because of short circuit
# with matching host to force
self.mox.ReplayAll()
result = fake_host.passes_filters(filter_fns, filter_properties)
self.assertTrue(result)
def test_stat_consumption_from_compute_node(self):
stats = [
dict(key='num_instances', value='5'),

125
nova/tests/test_filters.py Normal file
View File

@ -0,0 +1,125 @@
# Copyright 2012 OpenStack LLC. # All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Tests For Scheduler Host Filters.
"""
import inspect
import sys
from nova import filters
from nova import loadables
from nova import test
class Filter1(filters.BaseFilter):
"""Test Filter class #1."""
pass
class Filter2(filters.BaseFilter):
"""Test Filter class #2."""
pass
class FiltersTestCase(test.TestCase):
def test_filter_all(self):
filter_obj_list = ['obj1', 'obj2', 'obj3']
filter_properties = 'fake_filter_properties'
base_filter = filters.BaseFilter()
self.mox.StubOutWithMock(base_filter, '_filter_one')
base_filter._filter_one('obj1', filter_properties).AndReturn(True)
base_filter._filter_one('obj2', filter_properties).AndReturn(False)
base_filter._filter_one('obj3', filter_properties).AndReturn(True)
self.mox.ReplayAll()
result = base_filter.filter_all(filter_obj_list, filter_properties)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(list(result), ['obj1', 'obj3'])
def test_filter_all_recursive_yields(self):
"""Test filter_all() allows generators from previous filter_all()s."""
# filter_all() yields results. We want to make sure that we can
# call filter_all() with generators returned from previous calls
# to filter_all().
filter_obj_list = ['obj1', 'obj2', 'obj3']
filter_properties = 'fake_filter_properties'
base_filter = filters.BaseFilter()
self.mox.StubOutWithMock(base_filter, '_filter_one')
total_iterations = 200
# The order that _filter_one is going to get called gets
# confusing because we will be recursively yielding things..
# We are going to simulate the first call to filter_all()
# returning False for 'obj2'. So, 'obj1' will get yielded
# 'total_iterations' number of times before the first filter_all()
# call gets to processing 'obj2'. We then return 'False' for it.
# After that, 'obj3' gets yielded 'total_iterations' number of
# times.
for x in xrange(total_iterations):
base_filter._filter_one('obj1', filter_properties).AndReturn(True)
base_filter._filter_one('obj2', filter_properties).AndReturn(False)
for x in xrange(total_iterations):
base_filter._filter_one('obj3', filter_properties).AndReturn(True)
self.mox.ReplayAll()
objs = iter(filter_obj_list)
for x in xrange(total_iterations):
# Pass in generators returned from previous calls.
objs = base_filter.filter_all(objs, filter_properties)
self.assertTrue(inspect.isgenerator(objs))
self.assertEqual(list(objs), ['obj1', 'obj3'])
def test_get_filtered_objects(self):
filter_objs_initial = ['initial', 'filter1', 'objects1']
filter_objs_second = ['second', 'filter2', 'objects2']
filter_objs_last = ['last', 'filter3', 'objects3']
filter_properties = 'fake_filter_properties'
def _fake_base_loader_init(*args, **kwargs):
pass
self.stubs.Set(loadables.BaseLoader, '__init__',
_fake_base_loader_init)
filt1_mock = self.mox.CreateMock(Filter1)
filt2_mock = self.mox.CreateMock(Filter2)
self.mox.StubOutWithMock(sys.modules[__name__], 'Filter1',
use_mock_anything=True)
self.mox.StubOutWithMock(filt1_mock, 'filter_all')
self.mox.StubOutWithMock(sys.modules[__name__], 'Filter2',
use_mock_anything=True)
self.mox.StubOutWithMock(filt2_mock, 'filter_all')
Filter1().AndReturn(filt1_mock)
filt1_mock.filter_all(filter_objs_initial,
filter_properties).AndReturn(filter_objs_second)
Filter2().AndReturn(filt2_mock)
filt2_mock.filter_all(filter_objs_second,
filter_properties).AndReturn(filter_objs_last)
self.mox.ReplayAll()
filter_handler = filters.BaseFilterHandler(filters.BaseFilter)
filter_classes = [Filter1, Filter2]
result = filter_handler.get_filtered_objects(filter_classes,
filter_objs_initial,
filter_properties)
self.assertEqual(result, filter_objs_last)