diff --git a/doc/source/index.rst b/doc/source/index.rst index 66fba27..f05791f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -7,7 +7,7 @@ and extend your application by discovering and loading extensions ("*plugins*") at runtime. Many applications implement their own library for doing this, using ``__import__`` or :mod:`importlib`. stevedore avoids creating yet another extension -mechanism by building on top of `setuptools entry points`_. The code +mechanism by building on top of `entry points`_. The code for managing entry points tends to be repetitive, though, so stevedore provides manager classes for implementing common patterns for using dynamically loaded extensions. @@ -21,7 +21,7 @@ dynamically loaded extensions. install/index -.. _setuptools entry points: http://setuptools.readthedocs.io/en/latest/pkg_resources.html?#entry-points +.. _entry points: https://docs.python.org/3/library/importlib.metadata.html#entry-points .. rubric:: Indices and tables diff --git a/doc/source/user/patterns_loading.rst b/doc/source/user/patterns_loading.rst index 53860d5..9bd9b52 100644 --- a/doc/source/user/patterns_loading.rst +++ b/doc/source/user/patterns_loading.rst @@ -7,7 +7,7 @@ defines the API expected by the plugin code. Each entry point has a name, which does not have to be unique within a given namespace. The flexibility of this name management system makes it possible to use plugins in a variety of ways. The manager classes in stevedore wrap -:mod:`pkg_resources` to apply different rules matching the patterns +:mod:`importlib.metadata` to apply different rules matching the patterns described here. Drivers -- Single Name, Single Entry Point diff --git a/doc/source/user/tutorial/creating_plugins.rst b/doc/source/user/tutorial/creating_plugins.rst index 45991ed..eefe76b 100644 --- a/doc/source/user/tutorial/creating_plugins.rst +++ b/doc/source/user/tutorial/creating_plugins.rst @@ -109,7 +109,7 @@ for stevedore is located in ``stevedore.egg-info/entry_points.txt``: t2 = stevedore.tests.test_extension:FauxExtension t1 = stevedore.tests.test_extension:FauxExtension -:mod:`pkg_resources` uses the ``entry_points.txt`` file from all of +:mod:`importlib.metadata` uses the ``entry_points.txt`` file from all of the installed packages on the import path to find plugins. You should not modify these files, except by changing the list of entry points in ``setup.py``. diff --git a/doc/source/user/tutorial/index.rst b/doc/source/user/tutorial/index.rst index e3117fd..6a83544 100644 --- a/doc/source/user/tutorial/index.rst +++ b/doc/source/user/tutorial/index.rst @@ -20,7 +20,9 @@ application. * `Using setuptools entry points`_ * `Package Discovery and Resource Access using pkg_resources`_ * `Using Entry Points to Write Plugins | Pylons`_ + * `importlib.metadata`_ .. _Using setuptools entry points: http://reinout.vanrees.org/weblog/2010/01/06/zest-releaser-entry-points.html .. _Package Discovery and Resource Access using pkg_resources: http://pythonhosted.org/distribute/pkg_resources.html .. _Using Entry Points to Write Plugins | Pylons: http://docs.pylonsproject.org/projects/pylons-webframework/en/latest/advanced_pylons/entry_points_and_plugins.html +.. _importlib.metadata: https://docs.python.org/3/library/importlib.metadata.html#entry-points diff --git a/doc/source/user/tutorial/loading.rst b/doc/source/user/tutorial/loading.rst index 1a81084..80ef56c 100644 --- a/doc/source/user/tutorial/loading.rst +++ b/doc/source/user/tutorial/loading.rst @@ -81,7 +81,7 @@ with :meth:`map` in this example takes two arguments, the The :class:`Extension` passed :func:`format_data` is a class defined by stevedore that wraps the plugin. It includes the name of the -plugin, the :class:`EntryPoint` returned by :mod:`pkg_resources`, and +plugin, the :class:`EntryPoint` returned by :mod:`importlib.metadata`, and the plugin itself (the named object referenced by the plugin definition). When ``invoke_on_load`` is true, the :class:`Extension` will also have an :attr:`obj` attribute containing the value returned diff --git a/lower-constraints.txt b/lower-constraints.txt index 6e5ab3c..f126b0c 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -8,6 +8,7 @@ fixtures==3.0.0 gitdb==0.6.4 GitPython==1.0.1 imagesize==0.7.1 +importlib_metadata==1.7.0 Jinja2==2.10 linecache2==1.0.0 MarkupSafe==1.0 diff --git a/requirements.txt b/requirements.txt index 6de9f4e..06321a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ # process, which may cause wedges in the gate later. pbr!=2.1.0,>=2.0.0 # Apache-2.0 +importlib_metadata>=1.7.0;python_version<'3.8' # Apache-2.0 diff --git a/stevedore/_cache.py b/stevedore/_cache.py new file mode 100644 index 0000000..28a45fa --- /dev/null +++ b/stevedore/_cache.py @@ -0,0 +1,195 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Use a cache layer in front of entry point scanning.""" + +import errno +import glob +import hashlib +import itertools +import json +import logging +import os +import os.path +import struct +import sys + +try: + # For python 3.8 and later + import importlib.metadata as importlib_metadata +except ImportError: + # For everyone else + import importlib_metadata + + +log = logging.getLogger('stevedore._cache') + + +def _get_cache_dir(): + """Locate a platform-appropriate cache directory to use. + + Does not ensure that the cache directory exists. + """ + # Linux, Unix, AIX, etc. + if os.name == 'posix' and sys.platform != 'darwin': + # use ~/.cache if empty OR not set + base_path = os.environ.get("XDG_CACHE_HOME", None) \ + or os.path.expanduser('~/.cache') + return os.path.join(base_path, 'python-entrypoints') + + # Mac OS + elif sys.platform == 'darwin': + return os.path.expanduser('~/Library/Caches/Python Entry Points') + + # Windows (hopefully) + else: + base_path = os.environ.get('LOCALAPPDATA', None) \ + or os.path.expanduser('~\\AppData\\Local') + return os.path.join(base_path, 'Python Entry Points') + + +def _get_mtime(name): + try: + s = os.stat(name) + return s.st_mtime + except OSError as err: + if err.errno != errno.ENOENT: + raise + return -1.0 + + +def _ftobytes(f): + return struct.Struct('f').pack(f) + + +def _hash_settings_for_path(path): + """Return a hash and the path settings that created it. + """ + paths = [] + h = hashlib.sha256() + + # Tie the cache to the python interpreter, in case it is part of a + # virtualenv. + h.update(sys.executable.encode('utf-8')) + h.update(sys.prefix.encode('utf-8')) + + for entry in path: + mtime = _get_mtime(entry) + h.update(entry.encode('utf-8')) + h.update(_ftobytes(mtime)) + paths.append((entry, mtime)) + + for ep_file in itertools.chain( + glob.iglob(os.path.join(entry, + '*.dist-info', + 'entry_points.txt')), + glob.iglob(os.path.join(entry, + '*.egg-info', + 'entry_points.txt')) + ): + mtime = _get_mtime(ep_file) + h.update(ep_file.encode('utf-8')) + h.update(_ftobytes(mtime)) + paths.append((ep_file, mtime)) + + return (h.hexdigest(), paths) + + +def _build_cacheable_data(path): + real_groups = importlib_metadata.entry_points() + # Convert the namedtuple values to regular tuples + groups = {} + for name, group_data in real_groups.items(): + existing = set() + members = [] + groups[name] = members + for ep in group_data: + # Filter out duplicates that can occur when testing a + # package that provides entry points using tox, where the + # package is installed in the virtualenv that tox builds + # and is present in the path as '.'. + item = ep[:] # convert namedtuple to tuple + if item in existing: + continue + existing.add(item) + members.append(item) + return { + 'groups': groups, + 'sys.executable': sys.executable, + 'sys.prefix': sys.prefix, + } + + +class Cache: + + def __init__(self, cache_dir=None): + if cache_dir is None: + cache_dir = _get_cache_dir() + self._dir = cache_dir + self._internal = {} + + def _get_data_for_path(self, path): + if path is None: + path = sys.path + + internal_key = tuple(path) + if internal_key in self._internal: + return self._internal[internal_key] + + digest, path_values = _hash_settings_for_path(path) + filename = os.path.join(self._dir, digest) + try: + log.debug('reading %s', filename) + with open(filename, 'r') as f: + data = json.load(f) + except (IOError, json.JSONDecodeError): + data = _build_cacheable_data(path) + data['path_values'] = path_values + try: + log.debug('writing to %s', filename) + os.makedirs(self._dir, exist_ok=True) + with open(filename, 'w') as f: + json.dump(data, f) + except (IOError, OSError): + # Could not create cache dir or write file. + pass + + self._internal[internal_key] = data + return data + + def get_group_all(self, group, path=None): + result = [] + data = self._get_data_for_path(path) + group_data = data.get('groups', {}).get(group, []) + for vals in group_data: + result.append(importlib_metadata.EntryPoint(*vals)) + return result + + def get_group_named(self, group, path=None): + result = {} + for ep in self.get_group_all(group, path=path): + if ep.name not in result: + result[ep.name] = ep + return result + + def get_single(self, group, name, path=None): + for name, ep in self.get_group_named(group, path=path).items(): + if name == name: + return ep + raise ValueError('No entrypoint {!r} in group {!r}'.format( + group, name)) + + +_c = Cache() +get_group_all = _c.get_group_all +get_group_named = _c.get_group_named +get_single = _c.get_single diff --git a/stevedore/extension.py b/stevedore/extension.py index f5c2292..fd667d4 100644 --- a/stevedore/extension.py +++ b/stevedore/extension.py @@ -14,10 +14,9 @@ """ import operator -import pkg_resources - import logging +from . import _cache from .exception import NoMatches LOG = logging.getLogger(__name__) @@ -34,7 +33,7 @@ class Extension(object): :param name: The entry point name. :type name: str :param entry_point: The EntryPoint instance returned by - :mod:`pkg_resources`. + :mod:`entrypoints`. :type entry_point: EntryPoint :param plugin: The value returned by entry_point.load() :param obj: The object returned by ``plugin(*args, **kwds)`` if the @@ -55,8 +54,7 @@ class Extension(object): :return: A string representation of the target of the entry point in 'dotted.module:object' format. """ - return '%s:%s' % (self.entry_point.module_name, - self.entry_point.attrs[0]) + return self.entry_point.value class ExtensionManager(object): @@ -174,7 +172,7 @@ class ExtensionManager(object): """ if self.namespace not in self.ENTRY_POINT_CACHE: - eps = list(pkg_resources.iter_entry_points(self.namespace)) + eps = list(_cache.get_group_all(self.namespace)) self.ENTRY_POINT_CACHE[self.namespace] = eps return self.ENTRY_POINT_CACHE[self.namespace] @@ -222,7 +220,7 @@ class ExtensionManager(object): ep.require() plugin = ep.resolve() else: - plugin = ep.load(require=verify_requirements) + plugin = ep.load() if invoke_on_load: obj = plugin(*invoke_args, **invoke_kwds) else: diff --git a/stevedore/sphinxext.py b/stevedore/sphinxext.py index 26f01e0..11cf24c 100644 --- a/stevedore/sphinxext.py +++ b/stevedore/sphinxext.py @@ -34,29 +34,29 @@ def _simple_list(mgr): doc = _get_docstring(ext.plugin) or '\n' summary = doc.splitlines()[0].strip() yield('* %s -- %s' % (ext.name, summary), - ext.entry_point.module_name) + ext.entry_point.module) def _detailed_list(mgr, over='', under='-', titlecase=False): for name in sorted(mgr.names()): ext = mgr[name] if over: - yield (over * len(ext.name), ext.entry_point.module_name) + yield (over * len(ext.name), ext.entry_point.module) if titlecase: - yield (ext.name.title(), ext.entry_point.module_name) + yield (ext.name.title(), ext.entry_point.module) else: - yield (ext.name, ext.entry_point.module_name) + yield (ext.name, ext.entry_point.module) if under: - yield (under * len(ext.name), ext.entry_point.module_name) - yield ('\n', ext.entry_point.module_name) + yield (under * len(ext.name), ext.entry_point.module) + yield ('\n', ext.entry_point.module) doc = _get_docstring(ext.plugin) if doc: - yield (doc, ext.entry_point.module_name) + yield (doc, ext.entry_point.module) else: yield ('.. warning:: No documentation found in %s' % ext.entry_point, - ext.entry_point.module_name) - yield ('\n', ext.entry_point.module_name) + ext.entry_point.module) + yield ('\n', ext.entry_point.module) class ListPluginsDirective(rst.Directive): @@ -79,7 +79,7 @@ class ListPluginsDirective(rst.Directive): underline_style = self.options.get('underline-style', '=') def report_load_failure(mgr, ep, err): - LOG.warning(u'Failed to load %s: %s' % (ep.module_name, err)) + LOG.warning(u'Failed to load %s: %s' % (ep.module, err)) mgr = extension.ExtensionManager( namespace, diff --git a/stevedore/tests/test_driver.py b/stevedore/tests/test_driver.py index c568a3a..9230835 100644 --- a/stevedore/tests/test_driver.py +++ b/stevedore/tests/test_driver.py @@ -13,7 +13,12 @@ """Tests for stevedore.extension """ -import pkg_resources +try: + # For python 3.8 and later + import importlib.metadata as importlib_metadata +except ImportError: + # For everyone else + import importlib_metadata from stevedore import driver from stevedore import exception @@ -68,13 +73,15 @@ class TestCallback(utils.TestCase): extensions = [ extension.Extension( 'backend', - pkg_resources.EntryPoint.parse('backend = pkg1:driver'), + importlib_metadata.EntryPoint( + 'backend', 'pkg1:driver', 'backend'), 'pkg backend', None, ), extension.Extension( 'backend', - pkg_resources.EntryPoint.parse('backend = pkg2:driver'), + importlib_metadata.EntryPoint( + 'backend', 'pkg2:driver', 'backend'), 'pkg backend', None, ), diff --git a/stevedore/tests/test_extension.py b/stevedore/tests/test_extension.py index c06afb5..8fc9149 100644 --- a/stevedore/tests/test_extension.py +++ b/stevedore/tests/test_extension.py @@ -96,13 +96,13 @@ class TestCallback(utils.TestCase): def test_use_cache(self): # If we insert something into the cache of entry points, - # the manager should not have to call into pkg_resources + # the manager should not have to call into entrypoints # to find the plugins. cache = extension.ExtensionManager.ENTRY_POINT_CACHE cache['stevedore.test.faux'] = [] - with mock.patch('pkg_resources.iter_entry_points', + with mock.patch('stevedore._cache.get_group_all', side_effect= - AssertionError('called iter_entry_points')): + AssertionError('called get_group_all')): em = extension.ExtensionManager('stevedore.test.faux') names = em.names() self.assertEqual(names, []) @@ -235,9 +235,9 @@ class TestLoadRequirementsOldSetuptools(utils.TestCase): def test_verify_requirements(self): self.em._load_one_plugin(self.mock_ep, False, (), {}, verify_requirements=True) - self.mock_ep.load.assert_called_once_with(require=True) + self.mock_ep.load.assert_called_once_with() def test_no_verify_requirements(self): self.em._load_one_plugin(self.mock_ep, False, (), {}, verify_requirements=False) - self.mock_ep.load.assert_called_once_with(require=False) + self.mock_ep.load.assert_called_once_with() diff --git a/stevedore/tests/test_sphinxext.py b/stevedore/tests/test_sphinxext.py index 8b6a888..663a081 100644 --- a/stevedore/tests/test_sphinxext.py +++ b/stevedore/tests/test_sphinxext.py @@ -14,20 +14,25 @@ from unittest import mock +try: + # For python 3.8 and later + import importlib.metadata as importlib_metadata +except ImportError: + # For everyone else + import importlib_metadata + from stevedore import extension from stevedore import sphinxext from stevedore.tests import utils -import pkg_resources - def _make_ext(name, docstring): def inner(): pass inner.__doc__ = docstring - m1 = mock.Mock(spec=pkg_resources.EntryPoint) - m1.module_name = '%s_module' % name + m1 = mock.Mock(spec=importlib_metadata.EntryPoint) + m1.module = '%s_module' % name s = mock.Mock(return_value='ENTRY_POINT(%s)' % name) m1.__str__ = s return extension.Extension(name, m1, inner, None)