openstack/pbr
Code Issues Proposed changes

Add support for virtualenv 20.x

Browse Source 
virtualenv is undergoing a rewrite and has changed how the programmatic
API works [1]. Switch to the "new way".

While we're here, we also need to get Python 2.7 tests passing again.
That requires dropping support for upper-constraints and using our own,
limited local constraints based on supported Python versions. We also
need to migrate integration tests since those run with Python 3 now.
Update the scenarios for pip/setuptools in integration testing
similarly. Finally clean up the installation of all devstack repos as
requirements are managed different now. Instead of worrying about
syncing them we use constraints.

[1] https://github.com/pypa/virtualenv/issues/1585#issuecomment-585228492

Change-Id: I493e88985d2c4d09612fea4d20d8ffa20043a0cb
Signed-off-by: Stephen Finucane <sfinucan@redhat.com>
Depends-On: https://review.opendev.org/739014
This commit is contained in:
Stephen Finucane 2020-04-14 14:46:21 +01:00 committed by Clark Boylan
parent 9257b68435
commit 73c11c6267
7 changed files with 78 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lower-constraints.txt
View File

@ -22,7 +22,7 @@ pytz==2013.6
PyYAML==3.12
reno==2.5.0
requests==2.14.2
six==1.10.0
six==1.12.0
snowballstemmer==1.2.1
Sphinx==1.6.5
sphinxcontrib-apidoc==0.2.0
@ -34,4 +34,4 @@ testscenarios==0.4
testtools==2.2.0
traceback2==1.4.0
unittest2==1.1.0
virtualenv==14.0.6
virtualenv==20.0.3

7
pbr/tests/test_core.py
View File

@ -40,6 +40,7 @@
import glob
import os
import sys
import tarfile
import fixtures
@ -113,6 +114,12 @@ class TestCore(base.BaseTestCase):
def test_console_script_develop(self):
"""Test that we develop a non-pkg-resources console script."""
if sys.version_info < (3, 0):
self.skipTest(
'Fails with recent virtualenv due to '
'https://github.com/pypa/virtualenv/issues/1638'
)
if os.name == 'nt':
self.skipTest('Windows support is passthrough')

85
pbr/tests/test_integration.py
View File

@ -11,7 +11,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
try:
import configparser
except ImportError:
import ConfigParser as configparser
import os.path
import pkg_resources
import shlex
import sys
@ -77,19 +82,35 @@ class TestIntegration(base.BaseTestCase):
# We don't break these into separate tests because we'd need separate
# source dirs to isolate from side effects of running pip, and the
# overheads of setup would start to beat the benefits of parallelism.
self.useFixture(base.CapturedSubprocess(
'sync-req',
['python', 'update.py', os.path.join(REPODIR, self.short_name)],
cwd=os.path.join(REPODIR, 'requirements')))
self.useFixture(base.CapturedSubprocess(
'commit-requirements',
'git diff --quiet || git commit -amrequirements',
cwd=os.path.join(REPODIR, self.short_name), shell=True))
path = os.path.join(
self.useFixture(fixtures.TempDir()).path, 'project')
self.useFixture(base.CapturedSubprocess(
'clone',
['git', 'clone', os.path.join(REPODIR, self.short_name), path]))
path = os.path.join(REPODIR, self.short_name)
setup_cfg = os.path.join(path, 'setup.cfg')
project_name = pkg_resources.safe_name(self.short_name).lower()
# These projects should all have setup.cfg files but we'll be careful
if os.path.exists(setup_cfg):
config = configparser.ConfigParser()
config.read(setup_cfg)
if config.has_section('metadata'):
raw_name = config.get('metadata', 'name',
fallback='notapackagename')
# Technically we should really only need to use the raw
# name because all our projects should be good and use
# normalized names but they don't...
project_name = pkg_resources.safe_name(raw_name).lower()
constraints = os.path.join(REPODIR, 'requirements',
'upper-constraints.txt')
tmp_constraints = os.path.join(
self.useFixture(fixtures.TempDir()).path,
'upper-constraints.txt')
# We need to filter out the package we are installing to avoid
# conflicts with the constraints.
with open(constraints, 'r') as src:
with open(tmp_constraints, 'w') as dest:
for line in src:
constraint = line.split('===')[0]
if project_name != constraint:
dest.write(line)
pip_cmd = PIP_CMD + ['-c', tmp_constraints]
venv = self.useFixture(
test_packaging.Venv('sdist',
modules=['pip', 'wheel', PBRVERSION],
@ -105,7 +126,7 @@ class TestIntegration(base.BaseTestCase):
filename = os.path.join(
path, 'dist', os.listdir(os.path.join(path, 'dist'))[0])
self.useFixture(base.CapturedSubprocess(
'tarball', [python] + PIP_CMD + [filename]))
'tarball', [python] + pip_cmd + [filename]))
venv = self.useFixture(
test_packaging.Venv('install-git',
modules=['pip', 'wheel', PBRVERSION],
@ -113,7 +134,7 @@ class TestIntegration(base.BaseTestCase):
root = venv.path
python = venv.python
self.useFixture(base.CapturedSubprocess(
'install-git', [python] + PIP_CMD + ['git+file://' + path]))
'install-git', [python] + pip_cmd + ['git+file://' + path]))
if self.short_name == 'nova':
found = False
for _, _, filenames in os.walk(root):
@ -127,7 +148,7 @@ class TestIntegration(base.BaseTestCase):
root = venv.path
python = venv.python
self.useFixture(base.CapturedSubprocess(
'install-e', [python] + PIP_CMD + ['-e', path]))
'install-e', [python] + pip_cmd + ['-e', path]))
class TestInstallWithoutPbr(base.BaseTestCase):
@ -188,12 +209,16 @@ class TestInstallWithoutPbr(base.BaseTestCase):
class TestMarkersPip(base.BaseTestCase):
scenarios = [
('pip-1.5', {'modules': ['pip>=1.5,<1.6']}),
('pip-6.0', {'modules': ['pip>=6.0,<6.1']}),
('pip-latest', {'modules': ['pip']}),
('setuptools-EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8']}),
('setuptools-Trusty', {'modules': ['pip==1.5', 'setuptools==2.2']}),
('setuptools-minimum', {'modules': ['pip==1.5', 'setuptools==0.7.2']}),
('setuptools-Bionic', {
'modules': ['pip==9.0.1', 'setuptools==39.0.1']}),
('setuptools-Stretch', {
'modules': ['pip==9.0.1', 'setuptools==33.1.1']}),
('setuptools-EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}),
('setuptools-Buster', {
'modules': ['pip==18.1', 'setuptools==40.8.0']}),
('setuptools-Focal', {
'modules': ['pip==20.0.2', 'setuptools==45.2.0']}),
]
@testtools.skipUnless(
@ -240,25 +265,17 @@ class TestLTSSupport(base.BaseTestCase):
# These versions come from the versions installed from the 'virtualenv'
# command from the 'python-virtualenv' package.
scenarios = [
('EL7', {'modules': ['pip==1.4.1', 'setuptools==0.9.8'],
'py3support': True}), # And EPEL6
('Trusty', {'modules': ['pip==1.5', 'setuptools==2.2'],
'py3support': True}),
('Jessie', {'modules': ['pip==1.5.6', 'setuptools==5.5.1'],
'py3support': True}),
# Wheezy has pip1.1, which cannot be called with '-m pip'
# So we'll use a different version of pip here.
('WheezyPrecise', {'modules': ['pip==1.4.1', 'setuptools==0.6c11'],
'py3support': False})
('Bionic', {'modules': ['pip==9.0.1', 'setuptools==39.0.1']}),
('Stretch', {'modules': ['pip==9.0.1', 'setuptools==33.1.1']}),
('EL8', {'modules': ['pip==9.0.3', 'setuptools==39.2.0']}),
('Buster', {'modules': ['pip==18.1', 'setuptools==40.8.0']}),
('Focal', {'modules': ['pip==20.0.2', 'setuptools==45.2.0']}),
]
@testtools.skipUnless(
os.environ.get('PBR_INTEGRATION', None) == '1',
'integration tests not enabled')
def test_lts_venv_default_versions(self):
if (sys.version_info[0] == 3 and not self.py3support):
self.skipTest('This combination will not install with py3, '
'skipping test')
venv = self.useFixture(
test_packaging.Venv('setuptools', modules=self.modules))
bin_python = venv.python

3
pbr/tests/test_packaging.py
View File

@ -183,7 +183,8 @@ class Venv(fixtures.Fixture):
def _setUp(self):
path = self.useFixture(fixtures.TempDir()).path
virtualenv.create_environment(path, clear=True)
virtualenv.cli_run([path])
python = os.path.join(path, 'bin', 'python')
command = [python] + self.pip_cmd + ['-U']
if self.modules and len(self.modules) > 0:

7
test-requirements.txt
View File

@ -6,12 +6,13 @@ wheel>=0.32.0 # MIT
fixtures>=3.0.0 # Apache-2.0/BSD
hacking>=1.1.0,<1.2.0 # Apache-2.0
mock>=2.0.0 # BSD
six>=1.10.0 # MIT
stestr>=2.1.0 # Apache-2.0
six>=1.12.0 # MIT
stestr>=2.1.0,<3.0;python_version=='2.7' # Apache-2.0
stestr>=2.1.0;python_version>='3.0' # Apache-2.0
testresources>=2.0.0 # Apache-2.0/BSD
testscenarios>=0.4 # Apache-2.0/BSD
testtools>=2.2.0 # MIT
virtualenv>=14.0.6 # MIT
virtualenv>=20.0.3 # MIT
coverage!=4.4,>=4.0 # Apache-2.0
# optionally exposed by distutils commands

21
tools/integration.sh
View File

@ -97,10 +97,6 @@ name = test_project
[entry_points]
console_scripts =
test_cmd = test_project:main
[global]
setup-hooks =
pbr.hooks.setup_hook
EOF
cat <<EOF > setup.py
@ -115,18 +111,19 @@ from socket import error as SocketError
try:
setuptools.setup(
setup_requires=['pbr'],
pbr=True)
pbr=True,
)
except (SocketError, Timeout):
setuptools.setup(
setup_requires=['pbr'],
pbr=True)
pbr=True,
)
EOF
mkdir test_project
cat <<EOF > test_project/__init__.py
def main():
print "Test cmd"
print("Test cmd")
EOF
epvenv=$eptest/venv
@ -165,6 +162,8 @@ export REPODIR
export WHEELHOUSE
export OS_TEST_TIMEOUT=600
cd $REPODIR/pbr
tox -epy27 --notest
.tox/py27/bin/python -m pip install ${REPODIR}/requirements
tox -epy27 -- test_integration
mkvenv .venv
source .venv/bin/activate
pip install -r test-requirements.txt
pip install ${REPODIR}/requirements
stestr run --suppress-attachments test_integration

4
tox.ini
View File

@ -11,8 +11,9 @@ setenv =
OS_STDOUT_CAPTURE={env:OS_STDOUT_CAPTURE:1}
OS_STDERR_CAPTURE={env:OS_STDERR_CAPTURE:1}
OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:60}
# NOTE(stephenfin): pbr intentionally does not use constraints since we support
# a broader range of Python versions than OpenStack as a whole
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/test-requirements.txt
commands = stestr run --suppress-attachments {posargs}
@ -22,7 +23,6 @@ commands = flake8 {posargs}
[testenv:docs]
whitelist_externals = rm
deps =
-c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-r{toxinidir}/doc/requirements.txt
commands =
rm -rf doc/build doc/source/reference/api