Teach pbr to read extras and env markers

This adds support for reading extras from setup.cfg.

It also adds support for handling environment markers, both in the
extras section and in install_requires and in requirements.txt.

Change-Id: I6fd8276012e65f82934df9c374613b1ce6856b5a

doc/source/index.rst

@@ -123,6 +123,45 @@ version number used to install the package):
 Only the first file found is used to install the list of packages it
 contains.
 
+Extra requirements
+------------------
+
+Groups of optional dependencies (`"extra" requirements
+<https://www.python.org/dev/peps/pep-0426/#extras-optional-dependencies>`_)
+can be described in your setup.cfg, rather than needing to be added to
+setup.py. An example (which also demonstrates the use of environment
+markers) is shown below.
+
+Environment markers
+-------------------
+
+Environment markers are `conditional dependencies
+<https://www.python.org/dev/peps/pep-0426/#environment-markers>`_
+which can be added to the requirements (or to a group of extra
+requirements) automatically, depending on the environment the
+installer is running in. They can be added to requirements in the
+requirements file, or to extras definied in setup.cfg - but the format
+is slightly different for each.
+
+For ``requirements.txt``::
+
+    argparse; python=='2.6'
+
+will result in the package depending on ``argparse`` only if it's being
+installed into python2.6
+
+For extras specifed in setup.cfg, add an ``extras`` section. For
+instance, to create two groups of extra requirements with additional
+constraints on the environment, you can use::
+
+    [extras]
+    security =
+        aleph
+        bet :python_environment=='3.2'
+        gimel :python_environment=='2.7'
+    testing =
+        quux :python_environment=='2.7'
+
 long_description
 ----------------
 

pbr/tests/test_packaging.py

@@ -40,10 +40,14 @@
 
 import os
 import re
+import sys
 import tempfile
+import textwrap
 
 import fixtures
 import mock
+import pkg_resources
+import six
 import testscenarios
 from testtools import matchers
 
@@ -417,5 +421,57 @@ class TestVersions(base.BaseTestCase):
         self.assertEqual('1.3.0.0a1', version)
 
 
+class TestRequirementParsing(base.BaseTestCase):
+
+    def test_requirement_parsing(self):
+        tempdir = self.useFixture(fixtures.TempDir()).path
+        requirements = os.path.join(tempdir, 'requirements.txt')
+        with open(requirements, 'wt') as f:
+            f.write(textwrap.dedent(six.u("""\
+                bar
+                quux<1.0; python_version=='2.6'
+            """)))
+        setup_cfg = os.path.join(tempdir, 'setup.cfg')
+        with open(setup_cfg, 'wt') as f:
+            f.write(textwrap.dedent(six.u("""\
+                [metadata]
+                name = test_reqparse
+
+                [extras]
+                test =
+                    foo
+                    baz>3.2 :python_version=='2.7'
+            """)))
+        # pkg_resources.split_sections uses None as the title of an
+        # anonymous section instead of the empty string. Weird.
+        expected_requirements = {
+            None: ['bar'],
+            ":python_version=='2.6'": ['quux<1.0'],
+            "test:python_version=='2.7'": ['baz>3.2'],
+            "test": ['foo']
+        }
+        setup_py = os.path.join(tempdir, 'setup.py')
+        with open(setup_py, 'wt') as f:
+            f.write(textwrap.dedent(six.u("""\
+                #!/usr/bin/env python
+                import setuptools
+                setuptools.setup(
+                    setup_requires=['pbr'],
+                    pbr=True,
+                )
+            """)))
+
+        self._run_cmd(sys.executable, (setup_py, 'egg_info'),
+                      allow_fail=False, cwd=tempdir)
+        egg_info = os.path.join(tempdir, 'test_reqparse.egg-info')
+
+        requires_txt = os.path.join(egg_info, 'requires.txt')
+        with open(requires_txt, 'rt') as requires:
+            generated_requirements = dict(
+                pkg_resources.split_sections(requires))
+
+        self.assertEqual(expected_requirements, generated_requirements)
+
+
 def load_tests(loader, in_tests, pattern):
     return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)

pbr/tests/test_util.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import io
+import textwrap
+
+import six
+from six.moves import configparser
+import testscenarios
+
+from pbr.tests import base
+from pbr import util
+
+
+class TestExtrasRequireParsingScenarios(base.BaseTestCase):
+
+    scenarios = [
+        ('simple_extras', {
+            'config_text': """
+                [extras]
+                first =
+                    foo
+                    bar==1.0
+                second =
+                    baz>=3.2
+                    foo
+                """,
+            'expected_extra_requires': {'first': ['foo', 'bar==1.0'],
+                                        'second': ['baz>=3.2', 'foo']}
+        }),
+        ('with_markers', {
+            'config_text': """
+                [extras]
+                test =
+                    foo:python_version=='2.6'
+                    bar
+                    baz<1.6 :python_version=='2.6'
+                """,
+            'expected_extra_requires': {
+                "test:python_version=='2.6'": ['foo', 'baz<1.6'],
+                "test": ['bar']}}),
+        ('no_extras', {
+            'config_text': """
+            [metadata]
+            long_description = foo
+            """,
+            'expected_extra_requires':
+            {}
+        })]
+
+    def config_from_ini(self, ini):
+        config = {}
+        parser = configparser.SafeConfigParser()
+        ini = textwrap.dedent(six.u(ini))
+        parser.readfp(io.StringIO(ini))
+        for section in parser.sections():
+            config[section] = dict(parser.items(section))
+        return config
+
+    def test_extras_parsing(self):
+        config = self.config_from_ini(self.config_text)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+
+        self.assertEqual(self.expected_extra_requires,
+                         kwargs['extras_require'])
+
+
+def load_tests(loader, in_tests, pattern):
+    return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)

pbr/util.py

@@ -280,6 +280,10 @@ def setup_cfg_to_setup_kwargs(config):
 
     kwargs = {}
 
+    # Temporarily holds install_reqires and extra_requires while we
+    # parse env_markers.
+    all_requirements = {}
+
     for arg in D1_D2_SETUP_ARGS:
         if len(D1_D2_SETUP_ARGS[arg]) == 2:
             # The distutils field name is different than distutils2's.
@@ -326,6 +330,17 @@ def setup_cfg_to_setup_kwargs(config):
                 # setuptools
                 in_cfg_value = [_VERSION_SPEC_RE.sub(r'\1\2', pred)
                                 for pred in in_cfg_value]
+            if arg == 'install_requires':
+                # Split install_requires into package,env_marker tuples
+                # These will be re-assembled later
+                install_requires = []
+                requirement_pattern = '(?P<package>[^;]*);?(?P<env_marker>.*)$'
+                for requirement in in_cfg_value:
+                    m = re.match(requirement_pattern, requirement)
+                    requirement_package = m.group('package').strip()
+                    env_marker = m.group('env_marker').strip()
+                    install_requires.append((requirement_package,env_marker))
+                all_requirements[''] = install_requires
             elif arg == 'package_dir':
                 in_cfg_value = {'': in_cfg_value}
             elif arg in ('package_data', 'data_files'):
@@ -367,6 +382,50 @@ def setup_cfg_to_setup_kwargs(config):
 
         kwargs[arg] = in_cfg_value
 
+    # Transform requirements with embedded environment markers to
+    # setuptools' supported marker-per-requirement format.
+    #
+    # install_requires are treated as a special case of extras, before
+    # being put back in the expected place
+    #
+    # fred =
+    #     foo:marker
+    #     bar
+    # -> {'fred': ['bar'], 'fred:marker':['foo']}
+
+    if 'extras' in config:
+        requirement_pattern = '(?P<package>[^:]*):?(?P<env_marker>.*)$'
+        extras = config['extras']
+        for extra in extras:
+            extra_requirements = []
+            requirements = split_multiline(extras[extra])
+            for requirement in requirements:
+                m = re.match(requirement_pattern, requirement)
+                extras_value = m.group('package').strip()
+                env_marker = m.group('env_marker')
+                extra_requirements.append((extras_value,env_marker))
+            all_requirements[extra] = extra_requirements
+
+    # Transform the full list of requirements into:
+    # - install_requires, for those that have no extra and no
+    #   env_marker
+    # - named extras, for those with an extra name (which may include
+    #   an env_marker)
+    # - and as a special case, install_requires with an env_marker are
+    #   treated as named extras where the name is the empty string
+
+    extras_require = {}
+    for req_group in all_requirements:
+        for requirement, env_marker in all_requirements[req_group]:
+            if env_marker:
+                extras_key = '%s:%s' % (req_group, env_marker)
+            else:
+                extras_key = req_group
+            extras_require.setdefault(extras_key, []).append(requirement)
+
+    kwargs['install_requires'] = extras_require.pop('', [])
+    kwargs['extras_require'] = extras_require
+
     return kwargs
 
 

test-requirements.txt

@@ -5,6 +5,7 @@ hacking>=0.9.2,<0.10
 mock>=1.0
 python-subunit>=0.0.18
 sphinx>=1.1.2,<1.2
+six>=1.9.0
 testrepository>=0.0.18
 testresources>=0.2.4
 testscenarios>=0.4