From 73c11c626726ddadc0f43a57e5b7007236da38c5 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Tue, 14 Apr 2020 14:46:21 +0100 Subject: [PATCH] Add support for virtualenv 20.x virtualenv is undergoing a rewrite and has changed how the programmatic API works [1]. Switch to the "new way". While we're here, we also need to get Python 2.7 tests passing again. That requires dropping support for upper-constraints and using our own, limited local constraints based on supported Python versions. We also need to migrate integration tests since those run with Python 3 now. Update the scenarios for pip/setuptools in integration testing similarly. Finally clean up the installation of all devstack repos as requirements are managed different now. Instead of worrying about syncing them we use constraints. [1] https://github.com/pypa/virtualenv/issues/1585#issuecomment-585228492 Change-Id: I493e88985d2c4d09612fea4d20d8ffa20043a0cb Signed-off-by: Stephen Finucane Depends-On: https://review.opendev.org/739014 --- lower-constraints.txt | 4 +- pbr/tests/test_core.py | 7 +++ pbr/tests/test_integration.py | 85 +++++++++++++++++++++-------------- pbr/tests/test_packaging.py | 3 +- test-requirements.txt | 7 +-- tools/integration.sh | 21 +++++---- tox.ini | 4 +- 7 files changed, 78 insertions(+), 53 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index fc576a56..46ad80ba 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -22,7 +22,7 @@ pytz==2013.6 PyYAML==3.12 reno==2.5.0 requests==2.14.2 -six==1.10.0 +six==1.12.0 snowballstemmer==1.2.1 Sphinx==1.6.5 sphinxcontrib-apidoc==0.2.0 @@ -34,4 +34,4 @@ testscenarios==0.4 testtools==2.2.0 traceback2==1.4.0 unittest2==1.1.0 -virtualenv==14.0.6 +virtualenv==20.0.3 diff --git a/pbr/tests/test_core.py b/pbr/tests/test_core.py index ccd14aba..edb7c7b5 100644 --- a/pbr/tests/test_core.py +++ b/pbr/tests/test_core.py @@ -40,6 +40,7 @@ import glob import os +import sys import tarfile import fixtures @@ -113,6 +114,12 @@ class TestCore(base.BaseTestCase): def test_console_script_develop(self): """Test that we develop a non-pkg-resources console script.""" + if sys.version_info < (3, 0): + self.skipTest( + 'Fails with recent virtualenv due to ' + 'https://github.com/pypa/virtualenv/issues/1638' + ) + if os.name == 'nt': self.skipTest('Windows support is passthrough') diff --git a/pbr/tests/test_integration.py b/pbr/tests/test_integration.py index 8e96f21f..25473b05 100644 --- a/pbr/tests/test_integration.py +++ b/pbr/tests/test_integration.py @@ -11,7 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +try: + import configparser +except ImportError: + import ConfigParser as configparser import os.path +import pkg_resources import shlex import sys @@ -77,19 +82,35 @@ class TestIntegration(base.BaseTestCase): # We don't break these into separate tests because we'd need separate # source dirs to isolate from side effects of running pip, and the # overheads of setup would start to beat the benefits of parallelism. - self.useFixture(base.CapturedSubprocess( - 'sync-req', - ['python', 'update.py', os.path.join(REPODIR, self.short_name)], - cwd=os.path.join(REPODIR, 'requirements'))) - self.useFixture(base.CapturedSubprocess( - 'commit-requirements', - 'git diff --quiet || git commit -amrequirements', - cwd=os.path.join(REPODIR, self.short_name), shell=True)) - path = os.path.join( - self.useFixture(fixtures.TempDir()).path, 'project') - self.useFixture(base.CapturedSubprocess( - 'clone', - ['git', 'clone', os.path.join(REPODIR, self.short_name), path])) + path = os.path.join(REPODIR, self.short_name) + setup_cfg = os.path.join(path, 'setup.cfg') + project_name = pkg_resources.safe_name(self.short_name).lower() + # These projects should all have setup.cfg files but we'll be careful + if os.path.exists(setup_cfg): + config = configparser.ConfigParser() + config.read(setup_cfg) + if config.has_section('metadata'): + raw_name = config.get('metadata', 'name', + fallback='notapackagename') + # Technically we should really only need to use the raw + # name because all our projects should be good and use + # normalized names but they don't... + project_name = pkg_resources.safe_name(raw_name).lower() + constraints = os.path.join(REPODIR, 'requirements', + 'upper-constraints.txt') + tmp_constraints = os.path.join( + self.useFixture(fixtures.TempDir()).path, + 'upper-constraints.txt') + # We need to filter out the package we are installing to avoid + # conflicts with the constraints. + with open(constraints, 'r') as src: + with open(tmp_constraints, 'w') as dest: + for line in src: + constraint = line.split('===')[0] + if project_name != constraint: + dest.write(line) + pip_cmd = PIP_CMD + ['-c', tmp_constraints] + venv = self.useFixture( test_packaging.Venv('sdist', modules=['pip', 'wheel', PBRVERSION], @@ -105,7 +126,7 @@ class TestIntegration(base.BaseTestCase): filename = os.path.join( path, 'dist', os.listdir(os.path.join(path, 'dist'))[0]) self.useFixture(base.CapturedSubprocess( - 'tarball', [python] + PIP_CMD + [filename])) + 'tarball', [python] + pip_cmd + [filename])) venv = self.useFixture( test_packaging.Venv('install-git', modules=['pip', 'wheel', PBRVERSION], @@ -113,7 +134,7 @@ class TestIntegration(base.BaseTestCase): root = venv.path python = venv.python self.useFixture(base.CapturedSubprocess( - 'install-git', [python] + PIP_CMD + ['git+file://' + path])) + 'install-git', [python] + pip_cmd + ['git+file://' + path])) if self.short_name == 'nova': found = False for _, _, filenames in os.walk(root): @@ -127,7 +148,7 @@ class TestIntegration(base.BaseTestCase): root = venv.path python = venv.python self.useFixture(base.CapturedSubprocess( - 'install-e', [python] + PIP_CMD + ['-e', path])) + 'install-e', [python] + pip_cmd + ['-e', path])) class TestInstallWithoutPbr(base.BaseTestCase): @@ -188,12 +209,16 @@ class TestInstallWithoutPbr(base.BaseTestCase): class TestMarkersPip(base.BaseTestCase): scenarios = [ - ('pip-1.5', {'modules': ['pip>=1.5,<1.6']}), - ('pip-6.0', {'modules': ['pip>=6.0,<6.1']}), ('pip-latest', {'modules': ['pip']}), - ('setuptools-EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8']}), - ('setuptools-Trusty', {'modules': ['pip==1.5', 'setuptools==2.2']}), - ('setuptools-minimum', {'modules': ['pip==1.5', 'setuptools==0.7.2']}), + ('setuptools-Bionic', { + 'modules': ['pip==9.0.1', 'setuptools==39.0.1']}), + ('setuptools-Stretch', { + 'modules': ['pip==9.0.1', 'setuptools==33.1.1']}), + ('setuptools-EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}), + ('setuptools-Buster', { + 'modules': ['pip==18.1', 'setuptools==40.8.0']}), + ('setuptools-Focal', { + 'modules': ['pip==20.0.2', 'setuptools==45.2.0']}), ] @testtools.skipUnless( @@ -240,25 +265,17 @@ class TestLTSSupport(base.BaseTestCase): # These versions come from the versions installed from the 'virtualenv' # command from the 'python-virtualenv' package. scenarios = [ - ('EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8'], - 'py3support': True}), # And EPEL6 - ('Trusty', {'modules': ['pip==1.5', 'setuptools==2.2'], - 'py3support': True}), - ('Jessie', {'modules': ['pip==1.5.6', 'setuptools==5.5.1'], - 'py3support': True}), - # Wheezy has pip1.1, which cannot be called with '-m pip' - # So we'll use a different version of pip here. - ('WheezyPrecise', {'modules': ['pip==1.4.1', 'setuptools==0.6c11'], - 'py3support': False}) + ('Bionic', {'modules': ['pip==9.0.1', 'setuptools==39.0.1']}), + ('Stretch', {'modules': ['pip==9.0.1', 'setuptools==33.1.1']}), + ('EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}), + ('Buster', {'modules': ['pip==18.1', 'setuptools==40.8.0']}), + ('Focal', {'modules': ['pip==20.0.2', 'setuptools==45.2.0']}), ] @testtools.skipUnless( os.environ.get('PBR_INTEGRATION', None) == '1', 'integration tests not enabled') def test_lts_venv_default_versions(self): - if (sys.version_info[0] == 3 and not self.py3support): - self.skipTest('This combination will not install with py3, ' - 'skipping test') venv = self.useFixture( test_packaging.Venv('setuptools', modules=self.modules)) bin_python = venv.python diff --git a/pbr/tests/test_packaging.py b/pbr/tests/test_packaging.py index 07be5477..3a703194 100644 --- a/pbr/tests/test_packaging.py +++ b/pbr/tests/test_packaging.py @@ -183,7 +183,8 @@ class Venv(fixtures.Fixture): def _setUp(self): path = self.useFixture(fixtures.TempDir()).path - virtualenv.create_environment(path, clear=True) + virtualenv.cli_run([path]) + python = os.path.join(path, 'bin', 'python') command = [python] + self.pip_cmd + ['-U'] if self.modules and len(self.modules) > 0: diff --git a/test-requirements.txt b/test-requirements.txt index 09efec05..0358a796 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,12 +6,13 @@ wheel>=0.32.0 # MIT fixtures>=3.0.0 # Apache-2.0/BSD hacking>=1.1.0,<1.2.0 # Apache-2.0 mock>=2.0.0 # BSD -six>=1.10.0 # MIT -stestr>=2.1.0 # Apache-2.0 +six>=1.12.0 # MIT +stestr>=2.1.0,<3.0;python_version=='2.7' # Apache-2.0 +stestr>=2.1.0;python_version>='3.0' # Apache-2.0 testresources>=2.0.0 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD testtools>=2.2.0 # MIT -virtualenv>=14.0.6 # MIT +virtualenv>=20.0.3 # MIT coverage!=4.4,>=4.0 # Apache-2.0 # optionally exposed by distutils commands diff --git a/tools/integration.sh b/tools/integration.sh index 6c4dc16e..ff337ef0 100644 --- a/tools/integration.sh +++ b/tools/integration.sh @@ -97,10 +97,6 @@ name = test_project [entry_points] console_scripts = test_cmd = test_project:main - -[global] -setup-hooks = - pbr.hooks.setup_hook EOF cat < setup.py @@ -115,18 +111,19 @@ from socket import error as SocketError try: setuptools.setup( setup_requires=['pbr'], - pbr=True) + pbr=True, + ) except (SocketError, Timeout): setuptools.setup( setup_requires=['pbr'], - pbr=True) - + pbr=True, + ) EOF mkdir test_project cat < test_project/__init__.py def main(): - print "Test cmd" + print("Test cmd") EOF epvenv=$eptest/venv @@ -165,6 +162,8 @@ export REPODIR export WHEELHOUSE export OS_TEST_TIMEOUT=600 cd $REPODIR/pbr -tox -epy27 --notest -.tox/py27/bin/python -m pip install ${REPODIR}/requirements -tox -epy27 -- test_integration +mkvenv .venv +source .venv/bin/activate +pip install -r test-requirements.txt +pip install ${REPODIR}/requirements +stestr run --suppress-attachments test_integration diff --git a/tox.ini b/tox.ini index d48c7701..70f21278 100644 --- a/tox.ini +++ b/tox.ini @@ -11,8 +11,9 @@ setenv = OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:1} OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:1} OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:60} +# NOTE(stephenfin): pbr intentionally does not use constraints since we support +# a broader range of Python versions than OpenStack as a whole deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/test-requirements.txt commands = stestr run --suppress-attachments {posargs} @@ -22,7 +23,6 @@ commands = flake8 {posargs} [testenv:docs] whitelist_externals = rm deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt commands = rm -rf doc/build doc/source/reference/api