Switch from pkg_resources to importlib

setuptools seems to be in the process of deprecating pkg_resources.

Change-Id: I64f1434a5acab99057beb4f397adca85bdcc4ab6

swift/__init__.py

@@ -18,16 +18,30 @@ import sys
 import gettext
 import warnings
 
-import pkg_resources
+__version__ = None
 
+# First, try to get our version out of PKG-INFO. If we're installed,
+# this'll let us find our version without pulling in pbr. After all, if
+# we're installed on a system, we're not in a Git-managed source tree, so
+# pbr doesn't really buy us anything.
 try:
-    # First, try to get our version out of PKG-INFO. If we're installed,
+    import importlib.metadata
-    # this'll let us find our version without pulling in pbr. After all, if
+except ImportError:
-    # we're installed on a system, we're not in a Git-managed source tree, so
+    # python < 3.8
-    # pbr doesn't really buy us anything.
+    import pkg_resources
+    try:
         __version__ = __canonical_version__ = pkg_resources.get_provider(
             pkg_resources.Requirement.parse('swift')).version
-except pkg_resources.DistributionNotFound:
+    except pkg_resources.DistributionNotFound:
+        pass
+else:
+    try:
+        __version__ = __canonical_version__ = importlib.metadata.distribution(
+            'swift').version
+    except importlib.metadata.PackageNotFoundError:
+        pass
+
+if __version__ is None:
     # No PKG-INFO? We're probably running from a checkout, then. Let pbr do
     # its thing to figure out a version number.
     import pbr.version
@@ -35,6 +49,7 @@ except pkg_resources.DistributionNotFound:
     __version__ = _version_info.release_string()
     __canonical_version__ = _version_info.version_string()
 
+
 _localedir = os.environ.get('SWIFT_LOCALEDIR')
 _t = gettext.translation('swift', localedir=_localedir, fallback=True)
 

swift/common/middleware/s3api/etree.py

@@ -15,7 +15,17 @@
 
 import lxml.etree
 from copy import deepcopy
-from pkg_resources import resource_stream  # pylint: disable-msg=E0611
+try:
+    # importlib.resources was introduced in py37, but couldn't handle
+    # resources in subdirectories (which we use); files() added support
+    from importlib.resources import files
+    del files
+except ImportError:
+    # python < 3.9
+    from pkg_resources import resource_stream  # pylint: disable-msg=E0611
+else:
+    import importlib.resources
+    resource_stream = None
 import six
 
 from swift.common.utils import get_logger
@@ -70,7 +80,13 @@ def fromstring(text, root_tag=None, logger=None):
         # validate XML
         try:
             path = 'schema/%s.rng' % camel_to_snake(root_tag)
-            with resource_stream(__name__, path) as rng:
+            if resource_stream:
+                # python < 3.9
+                stream = resource_stream(__name__, path)
+            else:
+                stream = importlib.resources.files(
+                    __name__.rsplit('.', 1)[0]).joinpath(path).open('rb')
+            with stream as rng:
                 lxml.etree.RelaxNG(file=rng).assertValid(elem)
         except IOError as e:
             # Probably, the schema file doesn't exist.

swift/common/utils.py

@@ -58,7 +58,12 @@ import eventlet.debug
 import eventlet.greenthread
 import eventlet.patcher
 import eventlet.semaphore
-import pkg_resources
+try:
+    import importlib.metadata
+    pkg_resources = None
+except ImportError:
+    # python < 3.8
+    import pkg_resources
 from eventlet import GreenPool, sleep, Timeout
 from eventlet.event import Event
 from eventlet.green import socket, threading
@@ -6415,8 +6420,20 @@ def load_pkg_resource(group, uri):
 
     if scheme != 'egg':
         raise TypeError('Unhandled URI scheme: %r' % scheme)
+
+    if pkg_resources:
+        # python < 3.8
         return pkg_resources.load_entry_point(dist, group, name)
 
+    # May raise importlib.metadata.PackageNotFoundError
+    meta = importlib.metadata.distribution(dist)
+
+    entry_points = [ep for ep in meta.entry_points
+                    if ep.group == group and ep.name == name]
+    if not entry_points:
+        raise ImportError("Entry point %r not found" % ((group, name),))
+    return entry_points[0].load()
+
 
 class PipeMutex(object):
     """

test/unit/common/test_utils.py

@@ -4801,6 +4801,9 @@ cluster_dfw1 = http://dfw1.host/v1/
                        'X-Backend-Redirect-Timestamp': '-1'})
         self.assertIn('Invalid timestamp', str(exc))
 
+    @unittest.skipIf(sys.version_info >= (3, 8),
+                     'pkg_resources loading is only available on python 3.7 '
+                     'and earlier')
     @mock.patch('pkg_resources.load_entry_point')
     def test_load_pkg_resource(self, mock_driver):
         tests = {
@@ -4824,6 +4827,59 @@ cluster_dfw1 = http://dfw1.host/v1/
             utils.load_pkg_resource(*args)
         self.assertEqual("Unhandled URI scheme: 'nog'", str(cm.exception))
 
+    @unittest.skipIf(sys.version_info < (3, 8),
+                     'importlib loading is only available on python 3.8 '
+                     'and later')
+    @mock.patch('importlib.metadata.distribution')
+    def test_load_pkg_resource_importlib(self, mock_driver):
+        import importlib.metadata
+        repl_obj = object()
+        ec_obj = object()
+        other_obj = object()
+        mock_driver.return_value.entry_points = [
+            importlib.metadata.EntryPoint(group='swift.diskfile',
+                                          name='replication.fs',
+                                          value=repl_obj),
+            importlib.metadata.EntryPoint(group='swift.diskfile',
+                                          name='erasure_coding.fs',
+                                          value=ec_obj),
+            importlib.metadata.EntryPoint(group='swift.section',
+                                          name='thing.other',
+                                          value=other_obj),
+        ]
+        for ep in mock_driver.return_value.entry_points:
+            ep.load = lambda ep=ep: ep.value
+        tests = {
+            ('swift.diskfile', 'egg:swift#replication.fs'): repl_obj,
+            ('swift.diskfile', 'egg:swift#erasure_coding.fs'): ec_obj,
+            ('swift.section', 'egg:swift#thing.other'): other_obj,
+            ('swift.section', 'swift#thing.other'): other_obj,
+            ('swift.section', 'thing.other'): other_obj,
+        }
+        for args, expected in tests.items():
+            self.assertIs(expected, utils.load_pkg_resource(*args))
+            self.assertEqual(mock_driver.mock_calls, [mock.call('swift')])
+            mock_driver.reset_mock()
+
+        with self.assertRaises(TypeError) as cm:
+            args = ('swift.diskfile', 'nog:swift#replication.fs')
+            utils.load_pkg_resource(*args)
+        self.assertEqual("Unhandled URI scheme: 'nog'", str(cm.exception))
+
+        with self.assertRaises(ImportError) as cm:
+            args = ('swift.diskfile', 'other.fs')
+            utils.load_pkg_resource(*args)
+        self.assertEqual(
+            "Entry point ('swift.diskfile', 'other.fs') not found",
+            str(cm.exception))
+
+        with self.assertRaises(ImportError) as cm:
+            args = ('swift.missing', 'thing.other')
+            utils.load_pkg_resource(*args)
+        self.assertEqual(
+            "Entry point ('swift.missing', 'thing.other') not found",
+            str(cm.exception))
+
     @with_tempdir
     def test_systemd_notify(self, tempdir):
         m_sock = mock.Mock(connect=mock.Mock(), sendall=mock.Mock())