Merge "Add an API for loading a data source"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
a6a6466035
|
|
@ -0,0 +1,54 @@
|
|||
# Copyright 2015 Canonical Ltd.
|
||||
# This file is part of cloud-init. See LICENCE file for license information.
|
||||
#
|
||||
# vi: ts=4 expandtab
|
||||
|
||||
"""Various base classes and implementations for finding *plugins*."""
|
||||
|
||||
import abc
|
||||
import pkgutil
|
||||
|
||||
import six
|
||||
|
||||
from cloudinit import logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class BaseModuleIterator(object):
|
||||
"""Base class for describing a *module iterator*
|
||||
|
||||
A module iterator is a class that's capable of listing
|
||||
modules or packages from a specific location, which are
|
||||
already loaded.
|
||||
"""
|
||||
|
||||
def __init__(self, search_paths):
|
||||
self._search_paths = search_paths
|
||||
|
||||
@abc.abstractmethod
|
||||
def list_modules(self):
|
||||
"""List all the modules that this finder knows about."""
|
||||
|
||||
|
||||
class PkgutilModuleIterator(BaseModuleIterator):
|
||||
"""A class based on the *pkgutil* module for discovering modules."""
|
||||
|
||||
@staticmethod
|
||||
def _find_module(finder, module):
|
||||
"""Delegate to the *finder* for finding the given module."""
|
||||
return finder.find_module(module).load_module(module)
|
||||
|
||||
def list_modules(self):
|
||||
"""List all modules that this class knows about."""
|
||||
for finder, name, _ in pkgutil.walk_packages(self._search_paths):
|
||||
try:
|
||||
module = self._find_module(finder, name)
|
||||
except ImportError:
|
||||
LOG.debug('Could not import the module %r using the '
|
||||
'search path %r', name, finder.path)
|
||||
continue
|
||||
|
||||
yield module
|
||||
|
|
@ -4,9 +4,18 @@
|
|||
# vi: ts=4 expandtab
|
||||
|
||||
import abc
|
||||
import itertools
|
||||
|
||||
import six
|
||||
|
||||
from cloudinit import exceptions
|
||||
from cloudinit import logging
|
||||
from cloudinit import sources
|
||||
from cloudinit.sources import strategy
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class APIResponse(object):
|
||||
"""Holds API response content
|
||||
|
|
@ -35,6 +44,64 @@ class APIResponse(object):
|
|||
return self.decoded_buffer
|
||||
|
||||
|
||||
class DataSourceLoader(object):
|
||||
"""Class for retrieving an available data source instance
|
||||
|
||||
:param names:
|
||||
A list of possible data source names, from which the loader
|
||||
should pick. This can be used to filter the data sources
|
||||
that can be found from outside of cloudinit control.
|
||||
|
||||
:param module_iterator:
|
||||
An instance of :class:`cloudinit.plugin_finder.BaseModuleIterator`,
|
||||
which is used to find possible modules where the data sources
|
||||
can be found.
|
||||
|
||||
:param strategies:
|
||||
An iterator of search strategy classes, where each strategy is capable
|
||||
of filtering the data sources that can be used by cloudinit.
|
||||
Possible strategies includes serial data source search or
|
||||
parallel data source or filtering data sources according to
|
||||
some criteria (only network data sources)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, names, module_iterator, strategies):
|
||||
self._names = names
|
||||
self._module_iterator = module_iterator
|
||||
self._strategies = strategies
|
||||
|
||||
@staticmethod
|
||||
def _implements_source_api(module):
|
||||
"""Check if the given module implements the data source API."""
|
||||
return hasattr(module, 'data_sources')
|
||||
|
||||
def _valid_modules(self):
|
||||
"""Return all the modules that are *valid*
|
||||
|
||||
Valid modules are those that implements a particular API
|
||||
for declaring the data sources it exports.
|
||||
"""
|
||||
modules = self._module_iterator.list_modules()
|
||||
return filter(self._implements_source_api, modules)
|
||||
|
||||
def all_data_sources(self):
|
||||
"""Get all the data source classes that this finder knows about."""
|
||||
return itertools.chain.from_iterable(
|
||||
module.data_sources()
|
||||
for module in self._valid_modules())
|
||||
|
||||
def valid_data_sources(self):
|
||||
"""Get the data sources that are valid for this run."""
|
||||
data_sources = self.all_data_sources()
|
||||
# Instantiate them before passing to the strategies.
|
||||
data_sources = (data_source() for data_source in data_sources)
|
||||
|
||||
for strategy_instance in self._strategies:
|
||||
data_sources = strategy_instance.search_data_sources(data_sources)
|
||||
return data_sources
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class BaseDataSource(object):
|
||||
"""Base class for the data sources."""
|
||||
|
|
@ -106,3 +173,41 @@ class BaseDataSource(object):
|
|||
|
||||
def is_password_set(self):
|
||||
"""Check if the password was already posted to the metadata service."""
|
||||
|
||||
|
||||
def get_data_source(names, module_iterator, strategies=None):
|
||||
"""Get an instance of any data source available.
|
||||
|
||||
:param names:
|
||||
A list of possible data source names, from which the loader
|
||||
should pick. This can be used to filter the data sources
|
||||
that can be found from outside of cloudinit control.
|
||||
|
||||
:param module_iterator:
|
||||
A subclass of :class:`cloudinit.plugin_finder.BaseModuleIterator`,
|
||||
which is used to find possible modules where the data sources
|
||||
can be found.
|
||||
|
||||
:param strategies:
|
||||
An iterator of search strategy classes, where each strategy is capable
|
||||
of filtering the data sources that can be used by cloudinit.
|
||||
"""
|
||||
if names:
|
||||
default_strategies = [strategy.FilterNameStrategy(names)]
|
||||
else:
|
||||
default_strategies = []
|
||||
if strategies is None:
|
||||
strategies = []
|
||||
|
||||
strategy_instances = [strategy_cls() for strategy_cls in strategies]
|
||||
strategies = default_strategies + strategy_instances
|
||||
|
||||
iterator = module_iterator(sources.__path__)
|
||||
loader = DataSourceLoader(names, iterator, strategies)
|
||||
valid_sources = loader.valid_data_sources()
|
||||
|
||||
data_source = next(valid_sources, None)
|
||||
if not data_source:
|
||||
raise exceptions.CloudInitError('No available data source found')
|
||||
|
||||
return data_source
|
||||
|
|
|
|||
|
|
@ -125,3 +125,8 @@ class HttpOpenStackSource(baseopenstack.BaseOpenStackSource):
|
|||
return False
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def data_sources():
|
||||
"""Get the data sources exported in this module."""
|
||||
return (HttpOpenStackSource,)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
# Copyright 2015 Canonical Ltd.
|
||||
# This file is part of cloud-init. See LICENCE file for license information.
|
||||
#
|
||||
# vi: ts=4 expandtab
|
||||
|
||||
import abc
|
||||
|
||||
import six
|
||||
|
||||
from cloudinit import logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class BaseSearchStrategy(object):
|
||||
"""Declare search strategies for data sources
|
||||
|
||||
A *search strategy* represents a decoupled way of choosing
|
||||
one or more data sources from a list of data sources.
|
||||
Each strategy can be used interchangeably and they can
|
||||
be composed. For instance, once can apply a filtering strategy
|
||||
over a parallel search strategy, which looks for the available
|
||||
data sources.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def search_data_sources(self, data_sources):
|
||||
"""Search the possible data sources for this strategy
|
||||
|
||||
The method should filter the data sources that can be
|
||||
considered *valid* for the given strategy.
|
||||
|
||||
:param data_sources:
|
||||
An iterator of data source instances, where the lookup
|
||||
will be done.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def is_datasource_available(data_source):
|
||||
"""Check if the given *data_source* is considered *available*
|
||||
|
||||
A data source is considered available if it can be loaded,
|
||||
but other strategies could implement their own behaviour.
|
||||
"""
|
||||
try:
|
||||
if data_source.load():
|
||||
return True
|
||||
except Exception:
|
||||
LOG.error("Failed to load data source %r", data_source)
|
||||
return False
|
||||
|
||||
|
||||
class FilterNameStrategy(BaseSearchStrategy):
|
||||
"""A strategy for filtering data sources by name
|
||||
|
||||
:param names:
|
||||
A list of strings, where each string is a name for a possible
|
||||
data source. Only the data sources that are in this list will
|
||||
be loaded and filtered.
|
||||
"""
|
||||
|
||||
def __init__(self, names=None):
|
||||
self._names = names
|
||||
super(FilterNameStrategy, self).__init__()
|
||||
|
||||
def search_data_sources(self, data_sources):
|
||||
return (source for source in data_sources
|
||||
if source.__class__.__name__ in self._names)
|
||||
|
||||
|
||||
class SerialSearchStrategy(BaseSearchStrategy):
|
||||
"""A strategy that chooses a data source in serial."""
|
||||
|
||||
def search_data_sources(self, data_sources):
|
||||
for data_source in data_sources:
|
||||
if self.is_datasource_available(data_source):
|
||||
yield data_source
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
# Copyright 2015 Canonical Ltd.
|
||||
# This file is part of cloud-init. See LICENCE file for license information.
|
||||
#
|
||||
# vi: ts=4 expandtab
|
||||
|
||||
import functools
|
||||
import string
|
||||
import types
|
||||
|
||||
from cloudinit import exceptions
|
||||
from cloudinit import plugin_finder
|
||||
from cloudinit.sources import base
|
||||
from cloudinit.sources import strategy
|
||||
from cloudinit import tests
|
||||
|
||||
|
||||
class TestDataSourceDiscovery(tests.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestDataSourceDiscovery, self).setUp()
|
||||
self._modules = None
|
||||
|
||||
@property
|
||||
def modules(self):
|
||||
if self._modules:
|
||||
return self._modules
|
||||
|
||||
class Module(types.ModuleType):
|
||||
def data_sources(self):
|
||||
return (self, )
|
||||
|
||||
def __call__(self):
|
||||
return self
|
||||
|
||||
@property
|
||||
def __class__(self):
|
||||
return self
|
||||
|
||||
modules = self._modules = list(map(Module, string.ascii_letters))
|
||||
return modules
|
||||
|
||||
@property
|
||||
def module_iterator(self):
|
||||
modules = self.modules
|
||||
|
||||
class ModuleIterator(plugin_finder.BaseModuleIterator):
|
||||
def list_modules(self):
|
||||
return modules + [None, "", 42]
|
||||
|
||||
return ModuleIterator(None)
|
||||
|
||||
def test_loader_api(self):
|
||||
# Test that the API of DataSourceLoader is sane
|
||||
loader = base.DataSourceLoader(
|
||||
names=[], module_iterator=self.module_iterator,
|
||||
strategies=[])
|
||||
|
||||
all_data_sources = list(loader.all_data_sources())
|
||||
valid_data_sources = list(loader.valid_data_sources())
|
||||
|
||||
self.assertEqual(all_data_sources, self.modules)
|
||||
self.assertEqual(valid_data_sources, self.modules)
|
||||
|
||||
def test_loader_strategies(self):
|
||||
class OrdStrategy(strategy.BaseSearchStrategy):
|
||||
def search_data_sources(self, data_sources):
|
||||
return filter(lambda source: ord(source.__name__) < 100,
|
||||
data_sources)
|
||||
|
||||
class NameStrategy(strategy.BaseSearchStrategy):
|
||||
def search_data_sources(self, data_sources):
|
||||
return (source for source in data_sources
|
||||
if source.__name__ in ('a', 'b', 'c'))
|
||||
|
||||
loader = base.DataSourceLoader(
|
||||
names=[], module_iterator=self.module_iterator,
|
||||
strategies=(OrdStrategy(), NameStrategy(), ))
|
||||
valid_data_sources = list(loader.valid_data_sources())
|
||||
|
||||
self.assertEqual(len(valid_data_sources), 3)
|
||||
self.assertEqual([source.__name__ for source in valid_data_sources],
|
||||
['a', 'b', 'c'])
|
||||
|
||||
def test_get_data_source_filtered_by_name(self):
|
||||
source = base.get_data_source(
|
||||
names=['a', 'c'],
|
||||
module_iterator=self.module_iterator.__class__)
|
||||
self.assertEqual(source.__name__, 'a')
|
||||
|
||||
def test_get_data_source_multiple_strategies(self):
|
||||
class ReversedStrategy(strategy.BaseSearchStrategy):
|
||||
def search_data_sources(self, data_sources):
|
||||
return reversed(list(data_sources))
|
||||
|
||||
source = base.get_data_source(
|
||||
names=['a', 'b', 'c'],
|
||||
module_iterator=self.module_iterator.__class__,
|
||||
strategies=(ReversedStrategy, ))
|
||||
|
||||
self.assertEqual(source.__name__, 'c')
|
||||
|
||||
def test_get_data_source_no_data_source(self):
|
||||
get_data_source = functools.partial(
|
||||
base.get_data_source,
|
||||
names=['totallymissing'],
|
||||
module_iterator=self.module_iterator.__class__)
|
||||
|
||||
exc = self.assertRaises(exceptions.CloudInitError,
|
||||
get_data_source)
|
||||
self.assertEqual(str(exc), 'No available data source found')
|
||||
|
||||
def test_get_data_source_no_name_filtering(self):
|
||||
source = base.get_data_source(
|
||||
names=[], module_iterator=self.module_iterator.__class__)
|
||||
self.assertEqual(source.__name__, 'a')
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
# Copyright 2015 Canonical Ltd.
|
||||
# This file is part of cloud-init. See LICENCE file for license information.
|
||||
#
|
||||
# vi: ts=4 expandtab
|
||||
|
||||
from cloudinit.sources import strategy
|
||||
from cloudinit import tests
|
||||
from cloudinit.tests.util import mock
|
||||
|
||||
|
||||
class TestStrategy(tests.TestCase):
|
||||
|
||||
def test_custom_strategy(self):
|
||||
class CustomStrategy(strategy.BaseSearchStrategy):
|
||||
|
||||
def search_data_sources(self, data_sources):
|
||||
# Return them in reverse order
|
||||
return list(reversed(data_sources))
|
||||
|
||||
data_sources = [mock.sentinel.first, mock.sentinel.second]
|
||||
instance = CustomStrategy()
|
||||
sources = instance.search_data_sources(data_sources)
|
||||
|
||||
self.assertEqual(sources, [mock.sentinel.second, mock.sentinel.first])
|
||||
|
||||
def test_is_datasource_available(self):
|
||||
class CustomStrategy(strategy.BaseSearchStrategy):
|
||||
def search_data_sources(self, _):
|
||||
pass
|
||||
|
||||
instance = CustomStrategy()
|
||||
good_source = mock.Mock()
|
||||
good_source.load.return_value = True
|
||||
bad_source = mock.Mock()
|
||||
bad_source.load.return_value = False
|
||||
|
||||
self.assertTrue(instance.is_datasource_available(good_source))
|
||||
self.assertFalse(instance.is_datasource_available(bad_source))
|
||||
|
||||
def test_filter_name_strategy(self):
|
||||
names = ['first', 'second', 'third']
|
||||
full_names = names + ['fourth', 'fifth']
|
||||
sources = [type(name, (object, ), {})() for name in full_names]
|
||||
instance = strategy.FilterNameStrategy(names)
|
||||
|
||||
sources = list(instance.search_data_sources(sources))
|
||||
|
||||
self.assertEqual(len(sources), 3)
|
||||
self.assertEqual([source.__class__.__name__ for source in sources],
|
||||
names)
|
||||
|
||||
def test_serial_search_strategy(self):
|
||||
def is_available(self, data_source):
|
||||
return data_source in available_sources
|
||||
|
||||
sources = [mock.sentinel.first, mock.sentinel.second,
|
||||
mock.sentinel.third, mock.sentinel.fourth]
|
||||
available_sources = [mock.sentinel.second, mock.sentinel.fourth]
|
||||
|
||||
with mock.patch('cloudinit.sources.strategy.BaseSearchStrategy.'
|
||||
'is_datasource_available', new=is_available):
|
||||
instance = strategy.SerialSearchStrategy()
|
||||
valid_sources = list(instance.search_data_sources(sources))
|
||||
|
||||
self.assertEqual(available_sources, valid_sources)
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
# Copyright 2015 Canonical Ltd.
|
||||
# This file is part of cloud-init. See LICENCE file for license information.
|
||||
#
|
||||
# vi: ts=4 expandtab
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from cloudinit import plugin_finder
|
||||
from cloudinit.tests import TestCase
|
||||
from cloudinit.tests import util
|
||||
|
||||
|
||||
class TestPkgutilModuleIterator(TestCase):
|
||||
|
||||
@staticmethod
|
||||
@contextlib.contextmanager
|
||||
def _create_tmpdir():
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
yield tmpdir
|
||||
finally:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _create_package(self):
|
||||
with self._create_tmpdir() as tmpdir:
|
||||
path = os.path.join(tmpdir, 'good.py')
|
||||
with open(path, 'w') as stream:
|
||||
stream.write('name = 42')
|
||||
|
||||
# Make sure this fails.
|
||||
bad = os.path.join(tmpdir, 'bad.py')
|
||||
with open(bad, 'w') as stream:
|
||||
stream.write('import missingmodule')
|
||||
|
||||
yield tmpdir
|
||||
|
||||
def test_pkgutil_module_iterator(self):
|
||||
logging_format = ("Could not import the module 'bad' "
|
||||
"using the search path %r")
|
||||
|
||||
with util.LogSnatcher('cloudinit.plugin_finder') as snatcher:
|
||||
with self._create_package() as tmpdir:
|
||||
expected_logging = logging_format % tmpdir
|
||||
iterator = plugin_finder.PkgutilModuleIterator([tmpdir])
|
||||
modules = list(iterator.list_modules())
|
||||
|
||||
self.assertEqual(len(modules), 1)
|
||||
module = modules[0]
|
||||
self.assertEqual(module.name, 42)
|
||||
self.assertEqual(len(snatcher.output), 1)
|
||||
self.assertEqual(snatcher.output[0], expected_logging)
|
||||
Loading…
Reference in New Issue