From 56778928303b74112a83e9208f107b9fa06f12e7 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Mon, 12 Nov 2012 23:33:01 +0000 Subject: [PATCH] Add module for loading specific classes This adds nova/loadables.py which contains code that is to be shared by host and cell scheduling filtering and weighing. Most of this code originated from nova/scheduler/filters/__init__.py (which was copied from nova/api extension loading). This makes it more generic so that it can be shared. Note that this functionality is quite different than the generic plugin manager that exists in openstack-common. That code could not be used here without some significant changes. It also seems pretty overkill for the functionality required by scheduling filtering and weighing. Change-Id: I1b217dc2bc2d1dc2235c8251318d06b46597e8f4 --- nova/loadables.py | 116 ++++++++++++++++++++ nova/tests/fake_loadables/__init__.py | 27 +++++ nova/tests/fake_loadables/fake_loadable1.py | 44 ++++++++ nova/tests/fake_loadables/fake_loadable2.py | 39 +++++++ nova/tests/test_loadables.py | 113 +++++++++++++++++++ 5 files changed, 339 insertions(+) create mode 100644 nova/loadables.py create mode 100644 nova/tests/fake_loadables/__init__.py create mode 100644 nova/tests/fake_loadables/fake_loadable1.py create mode 100644 nova/tests/fake_loadables/fake_loadable2.py create mode 100644 nova/tests/test_loadables.py diff --git a/nova/loadables.py b/nova/loadables.py new file mode 100644 index 000000000000..0c930267ef3b --- /dev/null +++ b/nova/loadables.py @@ -0,0 +1,116 @@ +# Copyright (c) 2011-2012 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Generic Loadable class support. + +Meant to be used by such things as scheduler filters and weights where we +want to load modules from certain directories and find certain types of +classes within those modules. Note that this is quite different than +generic plugins and the pluginmanager code that exists elsewhere. + +Usage: + +Create a directory with an __init__.py with code such as: + +class SomeLoadableClass(object): + pass + + +class MyLoader(nova.loadables.BaseLoader) + def __init__(self): + super(MyLoader, self).__init__(SomeLoadableClass) + +If you create modules in the same directory and subclass SomeLoadableClass +within them, MyLoader().get_all_classes() will return a list +of such classes. +""" + +import inspect +import os +import sys + +from nova import exception +from nova.openstack.common import importutils + + +class BaseLoader(object): + def __init__(self, loadable_cls_type): + mod = sys.modules[self.__class__.__module__] + self.path = mod.__path__[0] + self.package = mod.__package__ + self.loadable_cls_type = loadable_cls_type + + def _is_correct_class(self, obj): + """Return whether an object is a class of the correct type and + is not prefixed with an underscore. + """ + return (inspect.isclass(obj) and + (not obj.__name__.startswith('_')) and + issubclass(obj, self.loadable_cls_type)) + + def _get_classes_from_module(self, module_name): + """Get the classes from a module that match the type we want.""" + classes = [] + module = importutils.import_module(module_name) + for obj_name in dir(module): + # Skip objects that are meant to be private. + if obj_name.startswith('_'): + continue + itm = getattr(module, obj_name) + if self._is_correct_class(itm): + classes.append(itm) + return classes + + def get_all_classes(self): + """Get the classes of the type we want from all modules found + in the directory that defines this class. + """ + classes = [] + for dirpath, dirnames, filenames in os.walk(self.path): + relpath = os.path.relpath(dirpath, self.path) + if relpath == '.': + relpkg = '' + else: + relpkg = '.%s' % '.'.join(relpath.split(os.sep)) + for fname in filenames: + root, ext = os.path.splitext(fname) + if ext != '.py' or root == '__init__': + continue + module_name = "%s%s.%s" % (self.package, relpkg, root) + mod_classes = self._get_classes_from_module(module_name) + classes.extend(mod_classes) + return classes + + def get_matching_classes(self, loadable_class_names): + """Get loadable classes from a list of names. Each name can be + a full module path or the full path to a method that returns + classes to use. The latter behavior is useful to specify a method + that returns a list of classes to use in a default case. + """ + classes = [] + for cls_name in loadable_class_names: + obj = importutils.import_class(cls_name) + if self._is_correct_class(obj): + classes.append(obj) + elif inspect.isfunction(obj): + # Get list of classes from a function + for cls in obj(): + classes.append(cls) + else: + error_str = 'Not a class of the correct type' + raise exception.ClassNotFound(class_name=cls_name, + exception=error_str) + return classes diff --git a/nova/tests/fake_loadables/__init__.py b/nova/tests/fake_loadables/__init__.py new file mode 100644 index 000000000000..824243347510 --- /dev/null +++ b/nova/tests/fake_loadables/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2012 OpenStack LLC. # All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Fakes For Loadable class handling. +""" + +from nova import loadables + + +class FakeLoadable(object): + pass + + +class FakeLoader(loadables.BaseLoader): + def __init__(self): + super(FakeLoader, self).__init__(FakeLoadable) diff --git a/nova/tests/fake_loadables/fake_loadable1.py b/nova/tests/fake_loadables/fake_loadable1.py new file mode 100644 index 000000000000..58f9704b30e1 --- /dev/null +++ b/nova/tests/fake_loadables/fake_loadable1.py @@ -0,0 +1,44 @@ +# Copyright 2012 OpenStack LLC. # All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Fake Loadable subclasses module #1 +""" + +from nova.tests import fake_loadables + + +class FakeLoadableSubClass1(fake_loadables.FakeLoadable): + pass + + +class FakeLoadableSubClass2(fake_loadables.FakeLoadable): + pass + + +class _FakeLoadableSubClass3(fake_loadables.FakeLoadable): + """Classes beginning with '_' will be ignored.""" + pass + + +class FakeLoadableSubClass4(object): + """Not a correct subclass.""" + + +def return_valid_classes(): + return [FakeLoadableSubClass1, FakeLoadableSubClass2] + + +def return_invalid_classes(): + return [FakeLoadableSubClass1, _FakeLoadableSubClass3, + FakeLoadableSubClass4] diff --git a/nova/tests/fake_loadables/fake_loadable2.py b/nova/tests/fake_loadables/fake_loadable2.py new file mode 100644 index 000000000000..3e365effc1e3 --- /dev/null +++ b/nova/tests/fake_loadables/fake_loadable2.py @@ -0,0 +1,39 @@ +# Copyright 2012 OpenStack LLC. # All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Fake Loadable subclasses module #2 +""" + +from nova.tests import fake_loadables + + +class FakeLoadableSubClass5(fake_loadables.FakeLoadable): + pass + + +class FakeLoadableSubClass6(fake_loadables.FakeLoadable): + pass + + +class _FakeLoadableSubClass7(fake_loadables.FakeLoadable): + """Classes beginning with '_' will be ignored.""" + pass + + +class FakeLoadableSubClass8(BaseException): + """Not a correct subclass.""" + + +def return_valid_class(): + return [FakeLoadableSubClass6] diff --git a/nova/tests/test_loadables.py b/nova/tests/test_loadables.py new file mode 100644 index 000000000000..6d16b9fa8b63 --- /dev/null +++ b/nova/tests/test_loadables.py @@ -0,0 +1,113 @@ +# Copyright 2012 OpenStack LLC. # All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Loadable class handling. +""" + +from nova import exception +from nova import test +from nova.tests import fake_loadables + + +class LoadablesTestCase(test.TestCase): + def setUp(self): + super(LoadablesTestCase, self).setUp() + self.fake_loader = fake_loadables.FakeLoader() + # The name that we imported above for testing + self.test_package = 'nova.tests.fake_loadables' + + def test_loader_init(self): + self.assertEqual(self.fake_loader.package, self.test_package) + # Test the path of the module + ending_path = '/' + self.test_package.replace('.', '/') + self.assertTrue(self.fake_loader.path.endswith(ending_path)) + self.assertEqual(self.fake_loader.loadable_cls_type, + fake_loadables.FakeLoadable) + + def _compare_classes(self, classes, expected): + class_names = [cls.__name__ for cls in classes] + self.assertEqual(set(class_names), set(expected)) + + def test_get_all_classes(self): + classes = self.fake_loader.get_all_classes() + expected_class_names = ['FakeLoadableSubClass1', + 'FakeLoadableSubClass2', + 'FakeLoadableSubClass5', + 'FakeLoadableSubClass6'] + self._compare_classes(classes, expected_class_names) + + def test_get_matching_classes(self): + prefix = self.test_package + test_classes = [prefix + '.fake_loadable1.FakeLoadableSubClass1', + prefix + '.fake_loadable2.FakeLoadableSubClass5'] + classes = self.fake_loader.get_matching_classes(test_classes) + expected_class_names = ['FakeLoadableSubClass1', + 'FakeLoadableSubClass5'] + self._compare_classes(classes, expected_class_names) + + def test_get_matching_classes_with_underscore(self): + prefix = self.test_package + test_classes = [prefix + '.fake_loadable1.FakeLoadableSubClass1', + prefix + '.fake_loadable2._FakeLoadableSubClass7'] + self.assertRaises(exception.ClassNotFound, + self.fake_loader.get_matching_classes, + test_classes) + + def test_get_matching_classes_with_wrong_type1(self): + prefix = self.test_package + test_classes = [prefix + '.fake_loadable1.FakeLoadableSubClass4', + prefix + '.fake_loadable2.FakeLoadableSubClass5'] + self.assertRaises(exception.ClassNotFound, + self.fake_loader.get_matching_classes, + test_classes) + + def test_get_matching_classes_with_wrong_type2(self): + prefix = self.test_package + test_classes = [prefix + '.fake_loadable1.FakeLoadableSubClass1', + prefix + '.fake_loadable2.FakeLoadableSubClass8'] + self.assertRaises(exception.ClassNotFound, + self.fake_loader.get_matching_classes, + test_classes) + + def test_get_matching_classes_with_one_function(self): + prefix = self.test_package + test_classes = [prefix + '.fake_loadable1.return_valid_classes', + prefix + '.fake_loadable2.FakeLoadableSubClass5'] + classes = self.fake_loader.get_matching_classes(test_classes) + expected_class_names = ['FakeLoadableSubClass1', + 'FakeLoadableSubClass2', + 'FakeLoadableSubClass5'] + self._compare_classes(classes, expected_class_names) + + def test_get_matching_classes_with_two_functions(self): + prefix = self.test_package + test_classes = [prefix + '.fake_loadable1.return_valid_classes', + prefix + '.fake_loadable2.return_valid_class'] + classes = self.fake_loader.get_matching_classes(test_classes) + expected_class_names = ['FakeLoadableSubClass1', + 'FakeLoadableSubClass2', + 'FakeLoadableSubClass6'] + self._compare_classes(classes, expected_class_names) + + def test_get_matching_classes_with_function_including_invalids(self): + # When using a method, no checking is done on valid classes. + prefix = self.test_package + test_classes = [prefix + '.fake_loadable1.return_invalid_classes', + prefix + '.fake_loadable2.return_valid_class'] + classes = self.fake_loader.get_matching_classes(test_classes) + expected_class_names = ['FakeLoadableSubClass1', + '_FakeLoadableSubClass3', + 'FakeLoadableSubClass4', + 'FakeLoadableSubClass6'] + self._compare_classes(classes, expected_class_names)