Add pyproject.toml support

Add the ability to read requirements and extras from pyproject.toml
files, eventually allowing us to move away from requirements.txt files
if we so choose.

Tests are reworked to test this new functionality, with some minor
cleanup to remove unused fixtures.

Change-Id: I3335b5faac72e2e6962d0930eef0e3b704820bbe
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>

.zuul.d/project-template.yaml

@@ -42,6 +42,7 @@
       - ^.*requirements-py[2,3].txt$
       - ^doc/requirements.txt$
       - ^lower-constraints.txt$
+      - ^pyproject.toml$
 
 - job:
     name: requirements-check-self

openstack_requirements/project.py

@@ -20,6 +20,41 @@ import errno
 import io
 import os
 
+try:
+    # Python 3.11+
+    import tomllib
+except ImportError:
+    # Python 3.10 and lower
+    import tomli as tomllib  # type: ignore
+
+
+def _read_pyproject_toml(root):
+    data = _read_raw(root, 'pyproject.toml')
+    if data is None:
+        return None
+
+    return tomllib.loads(data)
+
+
+def _read_pyproject_toml_requirements(root):
+    data = _read_pyproject_toml(root) or {}
+
+    # projects may not have PEP-621 project metadata
+    if 'project' not in data:
+        return None
+
+    return data['project'].get('dependencies')
+
+
+def _read_pyproject_toml_extras(root):
+    data = _read_pyproject_toml(root) or {}
+
+    # projects may not have PEP-621 project metadata
+    if 'project' not in data:
+        return None
+
+    return data['project'].get('optional-dependencies')
+
 
 def _read_setup_cfg_extras(root):
     data = _read_raw(root, 'setup.cfg')
@@ -28,10 +63,10 @@ def _read_setup_cfg_extras(root):
 
     c = configparser.ConfigParser()
     c.read_file(io.StringIO(data))
-    if c.has_section('extras'):
-        return dict(c.items('extras'))
+    if not c.has_section('extras'):
+        return None
 
-    return None
+    return dict(c.items('extras'))
 
 
 def _read_raw(root, filename):
@@ -60,6 +95,9 @@ def read(root):
     # Store requirements
     result['requirements'] = {}
 
+    if (data := _read_pyproject_toml_requirements(root)) is not None:
+        result['requirements']['pyproject.toml'] = data
+
     for filename in [
         'requirements.txt',
         'test-requirements.txt',
@@ -77,7 +115,11 @@ def read(root):
 
     # Store extras
     result['extras'] = {}
+
     if (data := _read_setup_cfg_extras(root)) is not None:
         result['extras']['setup.cfg'] = data
 
+    if (data := _read_pyproject_toml_extras(root)) is not None:
+        result['extras']['setup.cfg'] = data
+
     return result

openstack_requirements/tests/common.py

@@ -29,27 +29,44 @@ class Project(fixtures.Fixture):
     """A single project we can update."""
 
     def __init__(
-        self, req_path, setup_path, setup_cfg_path, test_req_path=None
+        self,
+        req_path=None,
+        setup_path=None,
+        setup_cfg_path=None,
+        test_req_path=None,
+        pyproject_toml_path=None,
     ):
         super().__init__()
         self._req_path = req_path
         self._setup_path = setup_path
         self._setup_cfg_path = setup_cfg_path
         self._test_req_path = test_req_path
+        self._pyproject_toml_path = pyproject_toml_path
 
     def setUp(self):
         super().setUp()
         self.root = self.useFixture(fixtures.TempDir()).path
+
         self.req_file = os.path.join(self.root, 'requirements.txt')
+        if self._req_path:
+            shutil.copy(self._req_path, self.req_file)
+
         self.setup_file = os.path.join(self.root, 'setup.py')
+        if self._setup_path:
+            shutil.copy(self._setup_path, self.setup_file)
+
         self.setup_cfg_file = os.path.join(self.root, 'setup.cfg')
+        if self._setup_cfg_path:
+            shutil.copy(self._setup_cfg_path, self.setup_cfg_file)
+
         self.test_req_file = os.path.join(self.root, 'test-requirements.txt')
-        shutil.copy(self._req_path, self.req_file)
-        shutil.copy(self._setup_path, self.setup_file)
-        shutil.copy(self._setup_cfg_path, self.setup_cfg_file)
         if self._test_req_path:
             shutil.copy(self._test_req_path, self.test_req_file)
 
+        self.pyproject_toml_file = os.path.join(self.root, 'pyproject.toml')
+        if self._pyproject_toml_path:
+            shutil.copy(self._pyproject_toml_path, self.pyproject_toml_file)
+
 
 project_fixture = Project(
     "openstack_requirements/tests/files/project.txt",
@@ -73,6 +90,9 @@ pbr_fixture = Project(
     "openstack_requirements/tests/files/pbr_setup.cfg",
     "openstack_requirements/tests/files/test-project.txt",
 )
+pep_518_fixture = Project(
+    pyproject_toml_path="openstack_requirements/tests/files/pyproject.toml",
+)
 
 
 class GlobalRequirements(fixtures.Fixture):
@@ -105,7 +125,4 @@ upper_constraints = requirement.parse(
 denylist = requirement.parse(
     open("openstack_requirements/tests/files/denylist.txt").read()
 )
-pbr_project = make_project(pbr_fixture)
 project_project = make_project(project_fixture)
-bad_project = make_project(bad_project_fixture)
-oslo_project = make_project(oslo_fixture)

openstack_requirements/tests/files/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["pbr>=6.1.1"]
+build-backend = "pbr.build"
+
+[project]
+name = "testproject"
+description = "OpenStack Test Project"
+authors = [
+    {name = "OpenStack", email = "openstack-discuss@lists.openstack.org"},
+]
+readme = {file = "README.rst", content-type = "text/x-rst"}
+license = {text = "Apache-2.0"}
+dynamic = ["version"]
+dependencies = [
+    "requests",
+    "debtcollector>=3.0",  # Apache-2.0
+]
+classifiers = [
+    "Environment :: OpenStack",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3",
+]
+
+[project.urls]
+Homepage = "https://docs.openstack.org/requirements"

openstack_requirements/tests/files/setup.cfg

@@ -5,14 +5,11 @@ description-file =
     README.rst
 author = OpenStack
 author-email = openstack-discuss@lists.openstack.org
-home-page = https://docs.openstack.org/requirements/latest/
+home-page = https://docs.openstack.org/requirements
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology
     Intended Audience :: System Administrators
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
-    Programming Language :: Python
-    Programming Language :: Python :: 2
-    Programming Language :: Python :: 2.7
-    Programming Language :: Python :: 2.6
+    Programming Language :: Python :: 3

openstack_requirements/tests/test_project.py

@@ -25,7 +25,16 @@ load_tests = testscenarios.load_tests_apply_scenarios
 
 
 class TestReadProject(testtools.TestCase):
-    def test_pbr(self):
+    def test_pyproject_toml(self):
+        root = self.useFixture(common.pep_518_fixture).root
+        proj = project.read(root)
+        self.assertEqual(proj['root'], root)
+        self.assertEqual(
+            list(sorted(proj['requirements'])),
+            ['pyproject.toml'],
+        )
+
+    def test_setup_cfg(self):
         root = self.useFixture(common.pbr_fixture).root
         proj = project.read(root)
         self.assertEqual(proj['root'], root)
@@ -34,7 +43,7 @@ class TestReadProject(testtools.TestCase):
             ['requirements.txt', 'test-requirements.txt'],
         )
 
-    def test_no_setup_py(self):
+    def test_empty(self):
         root = self.useFixture(fixtures.TempDir()).path
         proj = project.read(root)
         self.assertEqual(
@@ -48,7 +57,25 @@ class TestReadProject(testtools.TestCase):
 
 
 class TestProjectExtras(testtools.TestCase):
-    def test_smoke(self):
+    def test_pyproject_toml(self):
+        root = self.useFixture(fixtures.TempDir()).path
+        with open(os.path.join(root, 'pyproject.toml'), 'w') as fh:
+            fh.write(
+                textwrap.dedent("""
+                [project.optional-dependencies]
+                1 = [
+                  "foo",
+                ]
+                2 = [
+                  "foo", # fred
+                  "bar",
+                ]
+                """)
+            )
+        expected = {'1': ['foo'], '2': ['foo', 'bar']}
+        self.assertEqual(expected, project._read_pyproject_toml_extras(root))
+
+    def test_setup_cfg(self):
         root = self.useFixture(fixtures.TempDir()).path
         with open(os.path.join(root, 'setup.cfg'), 'w') as fh:
             fh.write(

requirements.txt

@@ -4,3 +4,4 @@ requests>=2.14.2 # Apache-2.0
 PyYAML>=3.12 # MIT
 beagle>=0.2.1 # Apache-2.0
 setuptools!=24.0.0,!=34.0.0,!=34.0.1,!=34.0.2,!=34.0.3,!=34.1.0,!=34.1.1,!=34.2.0,!=34.3.0,!=34.3.1,!=34.3.2,!=36.2.0,>=21.0.0 # PSF/ZPL
+tomli;python_version<'3.11' # MIT