openstack/rally
Code Issues Proposed changes

Remove usage of deprecated pkg_resources lib

Browse Source 
Signed-off-by: Andriy Kurilin <andr.kurilin@gmail.com>
Change-Id: I962a3d3ea7cfad5daedbb99f3f1c664af08e2f90
This commit is contained in:
Andriy Kurilin
2024-11-19 17:32:38 +01:00
parent dcba35a368
commit 5e83ecf74d
11 changed files with 176 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.rst
View File

@@ -25,6 +25,12 @@ Added
* CI jobs for checking compatibility with python 3.12
Changed
~~~~~~~
Replaced deprecated `pkg_resources` library with `importlib` & `packaging`
for plugins discovery and loading.
Removed
~~~~~~~

69
rally/common/plugin/discover.py
View File

@@ -14,10 +14,12 @@
# under the License.
import importlib
import importlib.metadata
import importlib.util
import os
import pkg_resources
import pkgutil
import sys
import typing as t
import rally
from rally.common import logging
@@ -50,7 +52,7 @@ def import_modules_from_package(package):
:param package: Full package name. For example: rally.plugins.openstack
"""
path = [os.path.dirname(rally.__file__), ".."] + package.split(".")
path = [os.path.dirname(rally.__file__), "..", *package.split(".")]
path = os.path.join(*path)
for root, dirs, files in os.walk(path):
for filename in files:
@@ -62,44 +64,50 @@ def import_modules_from_package(package):
sys.modules[module_name] = importlib.import_module(module_name)
def iter_entry_points(): # pragma: no cover
try:
# Python 3.10+
return importlib.metadata.entry_points(group="rally_plugins")
except TypeError:
# Python 3.8-3.9
return importlib.metadata.entry_points().get("rally_plugins", [])
def find_packages_by_entry_point():
"""Find all packages with rally_plugins entry-point"""
loaded_packages = []
packages = {}
for package in pkg_resources.working_set:
entry_map = package.get_entry_map("rally_plugins")
if not entry_map:
# this package doesn't have rally_plugins entry-point
for ep in iter_entry_points():
if ep.name not in ("path", "options"):
continue
if ep.dist.name not in packages:
packages[ep.dist.name] = {
"name": ep.dist.name,
"version": ep.dist.version
}
package_info = {}
if "path" in entry_map:
ep = entry_map["path"]
package_info["plugins_path"] = ep.module_name
if "options" in entry_map:
ep = entry_map["options"]
package_info["options"] = "%s:%s" % (
ep.module_name,
ep.attrs[0] if ep.attrs else "list_opts",
if ep.name == "path":
packages[ep.dist.name]["plugins_path"] = ep.value
packages[ep.dist.name]["plugins_path_ep"] = ep
elif ep.name == "options":
packages[ep.dist.name]["options"] = (
ep.value if ":" in ep.value else f"{ep.value}:list_opts"
)
if package_info:
package_info.update(
name=package.project_name,
version=package.version)
loaded_packages.append(package_info)
return loaded_packages
return list(packages.values())
def import_modules_by_entry_point(_packages=None):
def import_modules_by_entry_point(_packages: t.Union[list, None] = None):
"""Import plugins by entry-point 'rally_plugins'."""
loaded_packages = _packages or find_packages_by_entry_point()
if _packages is not None:
loaded_packages = _packages
else:
loaded_packages = find_packages_by_entry_point()
for package in loaded_packages:
if "plugins_path" in package:
em = pkg_resources.get_entry_map(package["name"])
ep = em["rally_plugins"]["path"]
ep = package["plugins_path_ep"]
try:
m = ep.load()
if hasattr(m, "__path__"):
@@ -109,11 +117,12 @@ def import_modules_by_entry_point(_packages=None):
prefix = m.__name__ + "."
for loader, name, _is_pkg in pkgutil.walk_packages(
path, prefix=prefix):
sys.modules[name] = importlib.import_module(name)
if name not in sys.modules:
sys.modules[name] = importlib.import_module(name)
except Exception as e:
msg = ("\t Failed to load plugins from module '%(module)s' "
"(package: '%(package)s')" %
{"module": ep.module_name,
{"module": ep.name,
"package": "%s %s" % (package["name"],
package["version"])})
if logging.is_debug():
@@ -145,7 +154,7 @@ def load_plugins(dir_or_file, depth=0):
if depth:
msg = "\t" + msg
LOG.info(msg)
module_name = os.path.splitext(plugin_file.split("/")[-1])[0]
module_name = os.path.splitext(os.path.basename(plugin_file))[0]
try:
spec = importlib.util.spec_from_file_location(
module_name, plugin_file)

16
rally/common/version.py
View File

@@ -15,6 +15,8 @@
from pbr import version as pbr_version
from rally.common.plugin import discover as rally_discover
RALLY_VENDOR = "OpenStack Foundation"
RALLY_PRODUCT = "OpenStack Rally"
RALLY_PACKAGE = None # OS distro package version suffix
@@ -35,14 +37,8 @@ def database_revision():
def plugins_versions():
"""Show packages version"""
import pkg_resources
packages = {}
for package in pkg_resources.working_set:
entry_map = package.get_entry_map("rally_plugins")
if not entry_map:
# this package doesn't have rally_plugins entry-point
continue
packages[package.project_name] = package.version
return packages
return dict(
(ep.dist.name, ep.dist.version)
for ep in rally_discover.iter_entry_points()
)

45
rally/verification/manager.py
View File

@@ -13,6 +13,8 @@
# under the License.
import abc
import importlib.metadata
import importlib.resources
import inspect
import io
import os
@@ -20,7 +22,8 @@ import re
import shutil
import sys
import pkg_resources
import packaging
import packaging.requirements
from rally.common.io import subunit_v2
from rally.common import logging
@@ -261,10 +264,12 @@ class VerifierManager(plugin.Plugin, metaclass=abc.ABCMeta):
LOG.debug("Initializing virtual environment in %s directory."
% self.venv_dir)
utils.check_output(["virtualenv", "-p", sys.executable, self.venv_dir],
cwd=self.repo_dir,
msg_on_err="Failed to initialize virtual env "
"in %s directory." % self.venv_dir)
utils.check_output(
["virtualenv", "-p", sys.executable, self.venv_dir],
cwd=self.repo_dir,
msg_on_err=f"Failed to initialize virtual env in "
f"{self.venv_dir} directory."
)
LOG.debug("Installing verifier in virtual environment.")
# NOTE(ylobankov): Use 'develop mode' installation to provide an
@@ -279,15 +284,27 @@ class VerifierManager(plugin.Plugin, metaclass=abc.ABCMeta):
reqs_file_path = reqs_file_path or os.path.join(self.repo_dir,
"requirements.txt")
with open(reqs_file_path) as f:
required_packages = [
p for p in f.read().split("\n")
if p.strip() and not p.startswith("#")
]
try:
pkg_resources.require(required_packages)
except (pkg_resources.DistributionNotFound,
pkg_resources.VersionConflict) as e:
raise VerifierSetupFailure(e.report(), verifier=self.verifier.name)
req_file_data = f.read()
for p in req_file_data.split("\n"):
if not p.strip() or p.startswith("#"):
continue
req = packaging.requirements.Requirement(p.split("#")[0].strip())
try:
version = importlib.metadata.distribution(req.name).version
except importlib.metadata.PackageNotFoundError:
raise VerifierSetupFailure(
f"The '{req}' distribution was not found, but "
f"is required by the application",
verifier=self.verifier.name
)
if not req.specifier.contains(version):
raise VerifierSetupFailure(
f"{req.name} {version} is installed but {req} is required",
verifier=self.verifier.name
)
def checkout(self, version):
"""Switch a verifier repo."""

1
requirements.txt
View File

@@ -21,3 +21,4 @@ python-subunit # Apache-2.0 or BSD
requests!=2.20.0,!=2.24.0 # Apache-2.0
SQLAlchemy>=2 # MIT
virtualenv!=16.3.0 # MIT
packaging!=20.5,!=20.6,!=20.7 # Apache Software License/BSD License

26
tests/ci/sync_requirements.py
View File

@@ -19,12 +19,12 @@ maximum allowed version).
"""
import collections
import importlib.metadata
import logging
import re
import sys
import textwrap
import pkg_resources
import requests
@@ -120,6 +120,10 @@ class PYPIPackage(object):
self._pypi_license = None
return self._license
@classmethod
def parse_line(cls, line):
raise NotImplementedError()
def __eq__(self, other):
return (isinstance(other, PYPIPackage)
and self.package_name == other.package_name)
@@ -262,7 +266,11 @@ class UpperConstraint(PYPIPackage):
return self.version
def parse_data(raw_data, include_comments=True, dependency_cls=Requirement):
def parse_data(
raw_data,
include_comments: bool = True,
dependency_cls: type[PYPIPackage] = Requirement
):
# first elem is None to simplify checks of last elem in requirements
requirements = [None]
for line in raw_data.split("\n"):
@@ -304,7 +312,7 @@ def parse_data(raw_data, include_comments=True, dependency_cls=Requirement):
for v in requirements if v)
def _fetch_from_gr(filename):
def _fetch_from_gr(filename: str) -> str:
"""Try to fetch data from OpenStack global-requirements repo"""
for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)):
url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + filename
@@ -316,7 +324,7 @@ def _fetch_from_gr(filename):
raise Exception("Unable to obtain %s" % filename)
def _write_requirements(filename, requirements):
def _write_requirements(filename: str, requirements: list | dict) -> None:
"""Saves requirements to file."""
if isinstance(requirements, dict):
requirements = requirements.values()
@@ -327,7 +335,7 @@ def _write_requirements(filename, requirements):
f.write("\n")
def sync_requirements():
def sync_requirements() -> None:
"""Synchronizes Rally requirements with OpenStack global-requirements."""
LOG.info("Obtaining global-requirements of OpenStack...")
raw_gr = _fetch_from_gr("global-requirements.txt")
@@ -350,7 +358,7 @@ def sync_requirements():
_write_requirements(file_name, requirements)
def update_upper_constraints():
def update_upper_constraints() -> None:
"""Obtains latest version of packages and put them to upper-constraints."""
LOG.info("Obtaining upper-constrains from OpenStack...")
raw_g_uc = _fetch_from_gr("upper-constraints.txt")
@@ -361,10 +369,10 @@ def update_upper_constraints():
include_comments=False,
dependency_cls=UpperConstraint)
our_uc = [UpperConstraint(package_name=p.project_name, version=p.version)
for p in pkg_resources.working_set
our_uc = [UpperConstraint(package_name=p.name, version=p.version)
for p in importlib.metadata.distributions()
# do not include the current package at u-c
if p.project_name != "rally"]
if p.name != "rally"]
for package in our_uc:
if package.package_name in global_uc:

15
tests/hacking/checks.py
View File

@@ -521,21 +521,6 @@ def check_raises(logical_line, filename, noqa=False):
"in docstrings.")
@core.flake8ext
def check_old_type_class(logical_line, noqa=False):
"""Use new-style Python classes
N355
"""
if noqa:
return
if re_old_type_class.search(logical_line):
yield (0, "N355 This class does not inherit from anything and thus "
"will be an old-style class by default. Try to inherit from "
"``object`` or another new-style class.")
@core.flake8ext
def check_datetime_alias(logical_line, noqa=False):
"""Ensure using ``dt`` as alias for ``datetime``

135
tests/unit/common/plugin/test_discover.py
View File

@@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import dataclasses
from unittest import mock
import uuid
@@ -144,24 +145,24 @@ class LoadExtraModulesTestCase(test.TestCase):
# TODO(olkonami): check exception is handled correct
discover.load_plugins("/somewhere/foo.py")
@mock.patch("%s.importlib" % DISCOVER)
@mock.patch("%s.importlib.import_module" % DISCOVER)
@mock.patch("%s.importlib.metadata.entry_points" % DISCOVER)
@mock.patch("%s.pkgutil.walk_packages" % DISCOVER)
@mock.patch("%s.pkg_resources" % DISCOVER)
def test_import_modules_by_entry_point(self, mock_pkg_resources,
mock_walk_packages, mock_importlib):
def test_import_modules_by_entry_point(
self, mock_walk_packages, mock_entry_points, mock_import_module
):
class Package(object):
def __init__(self, name, version, entry_map):
self.get_entry_map_called = False
self.project_name = name
self.version = version
self.entry_map = entry_map
@dataclasses.dataclass
class Dist:
name: str
version: str
def get_entry_map(self, group):
if self.get_entry_map_called is not False:
raise Exception("`get_entry_map` should be called once.")
self.get_entry_map_called = group
return self.entry_map.get(group, {})
@dataclasses.dataclass
class EntryPoint:
name: str
value: str
dist: Dist
load: mock.Mock
class LoadedPackage(object):
def __init__(self, name, path=None, file=None):
@@ -171,71 +172,63 @@ class LoadExtraModulesTestCase(test.TestCase):
if file is not None:
self.__file__ = file
class FakeEntryPoint(object):
def __init__(self, package_name=None, package_path=None,
package_file=None, module_name=None, attrs=None):
self.load = mock.Mock(return_value=LoadedPackage(
package_name, package_path, package_file))
self.module_name = module_name or str(uuid.uuid4())
self.attrs = attrs or tuple()
package1 = Dist(name="plugin1", version="1.2.3")
package2 = Dist(name="plugin2", version="1.2.4")
package3 = Dist(name="plugin3", version="1.2.5")
package4 = Dist(name="plugin4", version="1.2.6")
mock_pkg_resources.working_set = [
Package("foo", "0.1", entry_map={}),
Package("plugin1", "0.2",
entry_map={"rally_plugins": {
"path": FakeEntryPoint("plugin1", package_path="/foo"),
"foo": FakeEntryPoint("plugin1", package_path="/foo"),
"options": FakeEntryPoint(module_name="foo.bar",
attrs=("list_opts",))
}}),
Package("plugin2", "0.2.1",
entry_map={"rally_plugins": {
"path": FakeEntryPoint("plugin2",
package_path=None,
package_file="/bar")
}}),
Package("plugin3", "0.3",
entry_map={"rally_plugins": {
"path": FakeEntryPoint("plugin3",
package_path="/xxx",
package_file="/yyy")
}}),
Package("error", "6.6.6",
entry_map={"rally_plugins": {
"path": FakeEntryPoint("error")
}})
mock_entry_points.return_value = [
EntryPoint(
name="redundant",
dist=package1,
value="xxx",
load=mock.Mock()
),
EntryPoint(
dist=package1,
name="path",
value="some.module",
load=mock.Mock(return_value=LoadedPackage(name=package1.name,
path="/foo"))
),
EntryPoint(
dist=package2,
name="path",
value="another.module",
load=mock.Mock(return_value=LoadedPackage(name=package2.name,
file="/bar"))
),
EntryPoint(
dist=package3,
name="path",
value="yet_another.module",
load=mock.Mock(return_value=LoadedPackage(name=package3.name,
path="/xxx",
file="/yyy"))
),
EntryPoint(
dist=package4,
name="path",
value="error",
load=mock.Mock(return_value=LoadedPackage(name=package4.name))
)
]
def mock_get_entry_map(name, group=None):
self.assertIsNone(group)
for p in mock_pkg_resources.working_set:
if p.project_name == name:
return p.entry_map
mock_pkg_resources.get_entry_map.side_effect = mock_get_entry_map
# use random uuid to not have conflicts in sys.modules
packages = [[(mock.Mock(), str(uuid.uuid4()), None)] for i in range(3)]
mock_walk_packages.side_effect = packages
data = dict((p["name"], p)
for p in discover.import_modules_by_entry_point())
for package in mock_pkg_resources.working_set:
self.assertEqual("rally_plugins", package.get_entry_map_called)
entry_map = package.entry_map.get("rally_plugins", {})
for ep_name, ep in entry_map.items():
if ep_name == "path":
ep.load.assert_called_once_with()
self.assertIn(package.project_name, data)
self.assertEqual(package.version,
data[package.project_name]["version"])
else:
self.assertFalse(ep.load.called)
if ep_name == "options":
self.assertIn(package.project_name, data)
self.assertEqual(
"%s:%s" % (ep.module_name, ep.attrs[0]),
data[package.project_name]["options"])
mock_entry_points.assert_called_once_with(group="rally_plugins")
for ep in mock_entry_points.return_value:
if ep.name != "path":
self.assertFalse(ep.load.called)
else:
self.assertIn(ep.dist.name, data)
self.assertEqual(ep.dist.version,
data[ep.dist.name]["version"])
self.assertEqual([mock.call("/foo", prefix="plugin1."),
mock.call(["/bar"], prefix="plugin2."),

9
tests/unit/test_hacking.py
View File

@@ -352,15 +352,6 @@ class HackingTestCase(test.TestCase):
checkres = checks.check_objects_imports_in_cli(line, "./filename")
self.assertRaises(StopIteration, next, checkres)
@ddt.data(
"class Oldstype():",
"class Oldstyle:"
)
def test_check_old_type_class(self, line):
checkres = checks.check_old_type_class(line)
self.assertIsNotNone(next(checkres))
self.assertEqual([], list(checkres))
def test_check_datetime_alias(self):
lines = ["import datetime as date",
"import datetime",

4
tox.ini
View File

@@ -114,7 +114,6 @@ extension =
N351 = checks:check_no_constructor_data_struct
N352 = checks:check_dict_formatting_in_string
N354 = checks:check_raises
N355 = checks:check_old_type_class
N356 = checks:check_datetime_alias
N360 = checks:check_db_imports_in_cli
N361 = checks:check_objects_imports_in_cli
@@ -144,10 +143,9 @@ filterwarnings =
ignore: The frontend.Option class will be removed in Docutils 0.21 or later.:DeprecationWarning:
# python 3.10
ignore:The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives:DeprecationWarning:
ignore:pkg_resources is deprecated as an API:DeprecationWarning:
# pytest-cov & pytest-xdist
ignore:The --rsyncdir command line argument and rsyncdirs config variable are deprecated.:DeprecationWarning:
# python3.11 ?
ignore:datetime\.datetime\.utcnow\(\) is deprecated.*:DeprecationWarning:
ignore:datetime\.datetime\.utcfromtimestamp\(\) is deprecated.*:DeprecationWarning:
ignore:eventuletutils module is deprecated and will be removed\.:DeprecationWarning:
ignore:eventletutils module is deprecated and will be removed\.:DeprecationWarning:

33
upper-constraints.txt
View File

@@ -1,7 +1,5 @@
alembic===1.13.3
alembic===1.14.0
attrs===24.2.0
autocommand===2.2.2
backports.tarfile===1.2.0
bcrypt===4.0.1
certifi===2024.8.30
cffi===1.17.1
@@ -12,30 +10,23 @@ distlib===0.3.9
filelock===3.16.1
greenlet===3.1.1
idna===3.10
importlib-metadata===8.0.0
importlib-resources===6.4.0
inflect===7.3.1
importlib_metadata===8.5.0
iso8601===2.1.0
jaraco.collections===5.1.0
jaraco.context===6.0.1
jaraco.functools===4.1.0
jaraco.text===4.0.0
jinja2===3.1.4
Jinja2===3.1.4
jsonschema===4.19.2
jsonschema-specifications===2024.10.1
Mako===1.3.5
Mako===1.3.6
MarkupSafe===3.0.2
more-itertools===10.5.0
msgpack===1.1.0
netaddr===0.10.1
oslo.config===9.7.0
oslo.context===5.7.0
oslo.db===16.0.0
oslo.db===17.0.0
oslo.i18n===6.5.0
oslo.log===6.1.2
oslo.serialization===5.6.0
oslo.utils===7.3.0
packaging===24.1
oslo.utils===7.4.0
packaging===24.2
paramiko===3.5.0
pbr===6.1.0
pip===24.2
@@ -52,7 +43,7 @@ PyYAML===6.0.2
referencing===0.35.1
requests===2.32.3
rfc3986===2.0.0
rpds-py===0.20.0
rpds-py===0.21.0
setuptools===75.1.0
six===1.16.0
SQLAlchemy===2.0.36
@@ -60,13 +51,11 @@ stevedore===5.4.0
testresources===2.0.1
testscenarios===0.5.0
testtools===2.7.2
tomli===2.0.2
typeguard===4.3.0
typing-extensions===4.12.2
typing_extensions===4.12.2
tzdata===2024.2
urllib3===1.26.20
virtualenv===20.27.0
virtualenv===20.27.1
wcwidth===0.2.13
wheel===0.44.0
wrapt===1.16.0
zipp===3.20.2
zipp===3.21.0