openstack/requirements
Code Issues Proposed changes

provide more meaningful output on update.py

Browse Source 
In an attempt to make the proposal bot changes for requirements be
more meaningful make the update.py script return something that's more
user friendly.

Now by default the script will return the following to stdout on success:

Version change for: mox, mox3, testrepository, testtools
Updated project/test-requirements.txt:
    mox==0.5.3                     ->   mox>=0.5.3
    mox3==0.7.3                    ->   mox3>=0.7.0
    testrepository>=0.0.13         ->   testrepository>=0.0.17
    testtools>=0.9.27              ->   testtools>=0.9.32

Add a verbose flag that will provide much of the previous debug info.

Eliminate the giant dict dump of all the requirements files, as that
was largely impossible to make sense of, and rarely provided any
insight. During a devstack run you'll have git hashes for all the
trees, so can track down exact file contents if needed.

By making this the default stdout we can then capture and use this for
the git commit messages.

In order to make the test output matching human readable ignore the 80
column limit for the expected blocks via #noqa.

Change-Id: I85604db7bffafcd20bf3cc546fe3e5d7bda72193
This commit is contained in:
Sean Dague 2015-01-08 15:25:08 -05:00
parent f74bcaf4e0
commit 38b7490667
3 changed files with 114 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test-requirements.txt
View File

@ -1,6 +1,7 @@
# NOTE: These are requirements for testing the requirements project only
# See global-requirements for the actual requirements list
hacking>=0.9.4,<0.10
fixtures>=0.3.14
testrepository>=0.0.17
testscenarios>=0.4
testtools>=0.9.32,<0.9.35

90
tests/test_update.py
View File

@ -19,8 +19,8 @@ import os.path
import shutil
import subprocess
import sys
import tempfile
import fixtures
import testtools
@ -33,9 +33,7 @@ def _file_to_list(fname):
class UpdateTest(testtools.TestCase):
def setUp(self):
super(UpdateTest, self).setUp()
self.dir = tempfile.mkdtemp()
def _init_env(self):
self.project_dir = os.path.join(self.dir, "project")
self.bad_project_dir = os.path.join(self.dir, "bad_project")
self.oslo_dir = os.path.join(self.dir, "project_with_oslo")
@ -74,20 +72,28 @@ class UpdateTest(testtools.TestCase):
shutil.copy("tests/files/setup.cfg", self.oslo_setup_cfg_file)
shutil.copy("update.py", os.path.join(self.dir, "update.py"))
def _run_update(self):
# now go call update and see what happens
subprocess.check_output(
[sys.executable, "update.py", "project"])
subprocess.check_output([sys.executable, "update.py",
"project_with_oslo"])
def setUp(self):
super(UpdateTest, self).setUp()
self.dir = self.useFixture(fixtures.TempDir()).path
self._init_env()
# for convenience put us in the directory with the update.py
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
os.chdir(self.dir)
returncode = subprocess.call([sys.executable, "update.py", "project"])
self.assertEqual(returncode, 0)
returncode = subprocess.call([sys.executable, "update.py",
"project_with_oslo"])
self.assertEqual(returncode, 0)
def test_requirements(self):
self._run_update()
reqs = _file_to_list(self.req_file)
self.assertIn("jsonschema>=1.0.0,!=1.4.0,<2", reqs)
def test_project(self):
self._run_update()
reqs = _file_to_list(self.proj_file)
# ensure various updates take
self.assertIn("jsonschema>=1.0.0,!=1.4.0,<2", reqs)
@ -95,6 +101,7 @@ class UpdateTest(testtools.TestCase):
self.assertIn("SQLAlchemy>=0.7,<=0.7.99", reqs)
def test_requirements_header(self):
self._run_update()
_REQS_HEADER = [
'# The order of packages is significant, because pip processes '
'them in the order',
@ -106,6 +113,7 @@ class UpdateTest(testtools.TestCase):
self.assertEqual(_REQS_HEADER, reqs[:3])
def test_project_with_oslo(self):
self._run_update()
reqs = _file_to_list(self.oslo_file)
oslo_tar = ("-f http://tarballs.openstack.org/oslo.config/"
"oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3")
@ -114,6 +122,7 @@ class UpdateTest(testtools.TestCase):
self.assertNotIn("oslo.config>=1.1.0", reqs)
def test_test_project(self):
self._run_update()
reqs = _file_to_list(self.proj_test_file)
self.assertIn("testtools>=0.9.32", reqs)
self.assertIn("testrepository>=0.0.17", reqs)
@ -121,16 +130,19 @@ class UpdateTest(testtools.TestCase):
self.assertNotIn("sphinxcontrib-pecanwsme>=0.2", reqs)
def test_install_setup(self):
self._run_update()
setup_contents = _file_to_list(self.setup_file)
self.assertIn("# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO"
" - DO NOT EDIT", setup_contents)
def test_no_install_setup(self):
self._run_update()
setup_contents = _file_to_list(self.old_setup_file)
self.assertNotIn(
"# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO"
" - DO NOT EDIT", setup_contents)
# These are tests which don't need to run the project update in advance
def test_requirment_not_in_global(self):
returncode = subprocess.call([sys.executable, "update.py",
"bad_project"])
@ -150,3 +162,63 @@ class UpdateTest(testtools.TestCase):
self.assertEqual(returncode, 0)
reqs = _file_to_list(self.bad_proj_file)
self.assertIn("thisisnotarealdepedency", reqs)
# testing output
def test_non_verbose_output(self):
output = subprocess.check_output(
[sys.executable, "update.py", "project"])
expected = 'Version change for: greenlet, sqlalchemy, eventlet, pastedeploy, routes, webob, wsgiref, boto, kombu, pycrypto, python-swiftclient, lxml, jsonschema, python-keystoneclient\n' # noqa
expected += """Updated project/requirements.txt:
greenlet>=0.3.1 -> greenlet>=0.3.2
SQLAlchemy>=0.7.8,<=0.7.99 -> SQLAlchemy>=0.7,<=0.7.99
eventlet>=0.9.12 -> eventlet>=0.12.0
PasteDeploy -> PasteDeploy>=1.5.0
routes -> Routes>=1.12.3
WebOb>=1.2 -> WebOb>=1.2.3,<1.3
wsgiref -> wsgiref>=0.1.2
boto -> boto>=2.4.0
kombu>2.4.7 -> kombu>=2.4.8
pycrypto>=2.1.0alpha1 -> pycrypto>=2.6
python-swiftclient>=1.2,<2 -> python-swiftclient>=1.2
lxml -> lxml>=2.3
jsonschema -> jsonschema>=1.0.0,!=1.4.0,<2
python-keystoneclient>=0.2.0 -> python-keystoneclient>=0.4.1
Version change for: mox, mox3, testrepository, testtools
Updated project/test-requirements.txt:
mox==0.5.3 -> mox>=0.5.3
mox3==0.7.3 -> mox3>=0.7.0
testrepository>=0.0.13 -> testrepository>=0.0.17
testtools>=0.9.27 -> testtools>=0.9.32
"""
self.assertEqual(expected, output)
def test_verbose_output(self):
output = subprocess.check_output(
[sys.executable, "update.py", "-v", "project"])
expected = """Syncing project/requirements.txt
Version change for: greenlet, sqlalchemy, eventlet, pastedeploy, routes, webob, wsgiref, boto, kombu, pycrypto, python-swiftclient, lxml, jsonschema, python-keystoneclient\n""" # noqa
expected += """Updated project/requirements.txt:
greenlet>=0.3.1 -> greenlet>=0.3.2
SQLAlchemy>=0.7.8,<=0.7.99 -> SQLAlchemy>=0.7,<=0.7.99
eventlet>=0.9.12 -> eventlet>=0.12.0
PasteDeploy -> PasteDeploy>=1.5.0
routes -> Routes>=1.12.3
WebOb>=1.2 -> WebOb>=1.2.3,<1.3
wsgiref -> wsgiref>=0.1.2
boto -> boto>=2.4.0
kombu>2.4.7 -> kombu>=2.4.8
pycrypto>=2.1.0alpha1 -> pycrypto>=2.6
python-swiftclient>=1.2,<2 -> python-swiftclient>=1.2
lxml -> lxml>=2.3
jsonschema -> jsonschema>=1.0.0,!=1.4.0,<2
python-keystoneclient>=0.2.0 -> python-keystoneclient>=0.4.1
Syncing project/test-requirements.txt
Version change for: mox, mox3, testrepository, testtools
Updated project/test-requirements.txt:
mox==0.5.3 -> mox>=0.5.3
mox3==0.7.3 -> mox3>=0.7.0
testrepository>=0.0.13 -> testrepository>=0.0.17
testtools>=0.9.27 -> testtools>=0.9.32
Syncing setup.py
"""
self.assertEqual(expected, output)

37
update.py
View File

@ -33,6 +33,7 @@ import sys
from pip import req
VERBOSE = None
_setup_py_text = """#!/usr/bin/env python
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
@ -77,6 +78,21 @@ _REQS_HEADER = [
]
def verbose(msg):
if VERBOSE:
print(msg)
class Change(object):
def __init__(self, name, old, new):
self.name = name
self.old = old
self.new = new
def __repr__(self):
return "%-30.30s -> %s" % (self.old, self.new)
def _parse_pip(pip):
install_require = req.InstallRequirement.from_line(pip)
@ -128,13 +144,13 @@ def _parse_reqs(filename):
def _sync_requirements_file(source_reqs, dev_reqs, dest_path,
suffix, softupdate):
dest_reqs = _readlines(dest_path)
changes = []
# this is specifically for global-requirements gate jobs so we don't
# modify the git tree
if suffix:
dest_path = "%s.%s" % (dest_path, suffix)
print("Syncing %s" % dest_path)
verbose("Syncing %s" % dest_path)
with open(dest_path, 'w') as new_reqs:
@ -165,6 +181,8 @@ def _sync_requirements_file(source_reqs, dev_reqs, dest_path,
elif _functionally_equal(old_require, source_reqs[old_pip]):
new_reqs.write(old_line)
else:
changes.append(
Change(old_pip, old_require, source_reqs[old_pip]))
new_reqs.write("%s\n" % source_reqs[old_pip])
elif softupdate:
# under softupdate we pass through anything we don't
@ -187,6 +205,12 @@ def _sync_requirements_file(source_reqs, dev_reqs, dest_path,
print("'%s' is not in global-requirements.txt" % old_pip)
if os.getenv('NON_STANDARD_REQS', '0') != '1':
sys.exit(1)
# always print out what we did if we did a thing
if changes:
print("Version change for: %s" % ", ".join([x.name for x in changes]))
print("Updated %s:" % dest_path)
for change in changes:
print(" %s" % change)
def _copy_requires(suffix, softupdate, dest_dir):
@ -206,8 +230,6 @@ def _copy_requires(suffix, softupdate, dest_dir):
for dest in target_files:
dest_path = os.path.join(dest_dir, dest)
if os.path.exists(dest_path):
print("_sync_requirements_file(%s, %s, %s)" %
(source_reqs, dev_reqs, dest_path))
_sync_requirements_file(source_reqs, dev_reqs, dest_path,
suffix, softupdate)
@ -221,7 +243,7 @@ def _write_setup_py(dest_path):
has_pbr = 'pbr' in _read(target_setup_py)
is_pbr = 'name = pbr' in _read(setup_cfg)
if has_pbr and not is_pbr:
print("Syncing setup.py")
verbose("Syncing setup.py")
# We only want to sync things that are up to date with pbr mechanics
with open(target_setup_py, 'w') as setup_file:
setup_file.write(_setup_py_text)
@ -231,6 +253,8 @@ def main(options, args):
if len(args) != 1:
print("Must specify directory to update")
sys.exit(1)
global VERBOSE
VERBOSE = options.verbose
_copy_requires(options.suffix, options.softupdate, args[0])
_write_setup_py(args[0])
@ -242,5 +266,8 @@ if __name__ == "__main__":
parser.add_option("-s", "--soft-update", dest="softupdate",
action="store_true",
help="Pass through extra requirements without warning.")
parser.add_option("-v", "--verbose", dest="verbose",
action="store_true",
help="Add further verbosity to output")
(options, args) = parser.parse_args()
main(options, args)