openstack/releases
Code Issues Proposed changes

wrap subprocess to capture output in tests

Browse Source 
Provide some wrappers around subprocess functions to capture stderr
and stdout and divert it to the logs so that the test output is less
cluttered with the output of various git commands.

Change-Id: If36ef013aca498e3a0a9cc3a2b78d666775439ab
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2018-02-07 12:01:35 -05:00 committed by Sean McGinnis
parent aa8ee5a83a
commit bfd96f4fc0
4 changed files with 118 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
openstack_releases/gitutils.py
View File

@ -18,6 +18,7 @@ import os.path
import subprocess
from openstack_releases import links
from openstack_releases import processutils
# Disable warnings about insecure connections.
from requests.packages import urllib3
@ -31,7 +32,7 @@ CGIT_TAG_TEMPLATE = 'http://git.openstack.org/cgit/%s/tag/?h=%s'
def find_modified_deliverable_files():
"Return a list of files modified by the most recent commit."
results = subprocess.check_output(
results = processutils.check_output(
['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^']
).decode('utf-8')
filenames = [
@ -50,11 +51,11 @@ def commit_exists(workdir, repo, ref):
"""
try:
subprocess.check_output(
processutils.check_output(
['git', 'show', ref],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
except subprocess.CalledProcessError as err:
except processutils.CalledProcessError as err:
LOG.error('Could not find {}: {}'.format(ref, err))
return False
return True
@ -80,14 +81,14 @@ def ensure_basic_git_config(workdir, repo, settings):
for key, value in settings.items():
LOG.info('looking for git config {}'.format(key))
try:
existing = subprocess.check_output(
existing = processutils.check_output(
['git', 'config', '--get', key],
cwd=dest,
).decode('utf-8').strip()
LOG.info('using existing setting of {}: {!r}'.format(key, existing))
except subprocess.CalledProcessError:
except processutils.CalledProcessError:
LOG.info('updating setting of {} to {!r}'.format(key, value))
subprocess.check_call(
processutils.check_call(
['git', 'config', key, value],
cwd=dest,
)
@ -106,8 +107,7 @@ def clone_repo(workdir, repo, ref=None, branch=None):
if branch:
cmd.extend(['--branch', branch])
cmd.append(repo)
LOG.info(' '.join(cmd))
subprocess.check_call(cmd)
processutils.check_call(cmd)
dest = os.path.join(workdir, repo)
return dest
@ -131,13 +131,13 @@ def sha_for_tag(workdir, repo, version):
"""
# git log 2.3.11 -n 1 --pretty=format:%H
try:
actual_sha = subprocess.check_output(
actual_sha = processutils.check_output(
['git', 'log', str(version), '-n', '1', '--pretty=format:%H'],
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8')
actual_sha = actual_sha.strip()
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.info('ERROR getting SHA for tag %r: %s [%s]',
version, e, e.output.strip())
actual_sha = ''
@ -158,13 +158,13 @@ def stable_branch_exists(workdir, repo, series):
remote_match = 'remotes/origin/stable/%s' % series
try:
containing_branches = _filter_branches(
subprocess.check_output(
processutils.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
)
return (remote_match in containing_branches)
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.error('failed checking for branch: %s [%s]', e, e.output.strip())
return False
@ -186,7 +186,7 @@ def check_branch_sha(workdir, repo, series, sha):
remote_match = 'remotes/origin/stable/%s' % series
try:
containing_branches = _filter_branches(
subprocess.check_output(
processutils.check_output(
['git', 'branch', '-a', '--contains', sha],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
@ -203,7 +203,7 @@ def check_branch_sha(workdir, repo, series, sha):
# that series. Allow the release, as long as it is on the
# master branch.
all_branches = _filter_branches(
subprocess.check_output(
processutils.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
@ -223,7 +223,7 @@ def check_branch_sha(workdir, repo, series, sha):
LOG.debug('did not find SHA on %s or master or origin/master',
remote_match)
return False
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.error('failed checking SHA on branch: %s [%s]' % (e, e.output.strip()))
return False
@ -231,13 +231,13 @@ def check_branch_sha(workdir, repo, series, sha):
def check_ancestry(workdir, repo, old_version, sha):
"Check if the SHA is in the ancestry of the previous version."
try:
ancestors = subprocess.check_output(
ancestors = processutils.check_output(
['git', 'log', '--oneline', '--ancestry-path',
'%s..%s' % (old_version, sha)],
cwd=os.path.join(workdir, repo),
).decode('utf-8').strip()
return bool(ancestors)
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.error('failed checking ancestry: %s [%s]' % (e, e.output.strip()))
return False
@ -247,12 +247,12 @@ def get_latest_tag(workdir, repo, sha=None):
if sha is not None:
cmd.append(sha)
try:
return subprocess.check_output(
return processutils.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.warning('failed to retrieve latest tag: %s [%s]',
e, e.output.strip())
return None
@ -262,12 +262,12 @@ def add_tag(workdir, repo, tag, sha):
cmd = ['git', 'tag', '-m', 'temporary tag', tag, sha]
try:
LOG.info(' '.join(cmd))
return subprocess.check_output(
return processutils.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.warning('failed to add tag: %s [%s]',
e, e.output.strip())
return None
@ -275,7 +275,7 @@ def add_tag(workdir, repo, tag, sha):
def get_branches(workdir, repo):
try:
output = subprocess.check_output(
output = processutils.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
@ -300,7 +300,7 @@ def get_branches(workdir, repo):
continue
results.append(branch)
return results
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.error('failed to retrieve list of branches: %s [%s]',
e, e.output.strip())
return []
@ -308,7 +308,7 @@ def get_branches(workdir, repo):
def branches_containing(workdir, repo, ref):
try:
output = subprocess.check_output(
output = processutils.check_output(
['git', 'branch', '-r', '--contains', ref],
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
@ -319,7 +319,7 @@ def branches_containing(workdir, repo, ref):
for line in output.splitlines():
results.append(line.strip())
return results
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.error('failed to retrieve list of branches containing %s: %s [%s]',
ref, e, e.output.strip())
return []
@ -339,12 +339,12 @@ def get_branch_base(workdir, repo, branch):
'master',
]
try:
parents = subprocess.check_output(
parents = processutils.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.warning('failed to retrieve branch base: %s [%s]',
e, e.output.strip())
return None
@ -356,12 +356,12 @@ def get_branch_base(workdir, repo, branch):
'{}^^!'.format(parent),
]
try:
return subprocess.check_output(
return processutils.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except subprocess.CalledProcessError as e:
except processutils.CalledProcessError as e:
LOG.warning('failed to retrieve branch base: %s [%s]',
e, e.output.strip())
return None

82
openstack_releases/processutils.py Normal file
View File

@ -0,0 +1,82 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import subprocess
LOG = logging.getLogger(__name__)
from subprocess import CalledProcessError # noqa
def _multi_line_log(level, msg):
for line in msg.splitlines():
LOG.log(level, line)
def check_call(*popenargs, timeout=None, **kwargs):
# A variation of subprocess.check_call that captures and then
# logs the output of the command which makes it easier for tests
# to capture it.
kwargs['stdout'] = subprocess.PIPE
kwargs['stderr'] = subprocess.STDOUT
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
LOG.info('$ {}'.format(' '.join(cmd)))
completed = subprocess.run(*popenargs, **kwargs)
_multi_line_log(logging.INFO, completed.stdout.decode('utf-8'))
if completed.returncode:
raise subprocess.CalledProcessError(completed.returncode, cmd)
return 0
def check_output(*popenargs, timeout=None, **kwargs):
# A variation of subprocess.check_output that captures stderr and
# logs it instead of letting it go to the console directly to make
# it easier for tests to capture it.
# NOTE(dhellmann): copied from subprocess.py vv
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
if 'input' in kwargs and kwargs['input'] is None:
# Explicitly passing input=None was previously equivalent to passing an
# empty string. That is maintained here for backwards compatibility.
kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
# NOTE(dhellmann): end copied from subprocess.py ^^
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
LOG.info('$ {}'.format(' '.join(cmd)))
if 'stderr' not in kwargs:
kwargs['stderr'] = subprocess.PIPE
completed = subprocess.run(*popenargs,
stdout=subprocess.PIPE,
timeout=timeout,
check=True,
**kwargs)
if completed.stderr:
_multi_line_log(logging.WARNING, completed.stderr.decode('utf-8'))
return completed.stdout

9
openstack_releases/pythonutils.py
View File

@ -15,10 +15,11 @@
import logging
import os
import os.path
import subprocess
import requests
from openstack_releases import processutils
LOG = logging.getLogger(__name__)
@ -33,7 +34,7 @@ def get_sdist_name(workdir, repo):
# Use tox to set up a virtualenv so we can install the
# dependencies for the package. This only seems to be
# necessary for pbr, but...
subprocess.check_output(
processutils.check_output(
['tox', '-e', 'venv', '--notest'],
cwd=dest,
)
@ -44,10 +45,10 @@ def get_sdist_name(workdir, repo):
# Run it once and discard the result to ensure any setup_requires
# dependencies are installed.
cmd = [python, 'setup.py', '--name']
subprocess.check_output(cmd, cwd=dest)
processutils.check_output(cmd, cwd=dest)
# Run it again to get a clean version of the name.
print('Running: %s in %s' % (' '.join(cmd), dest))
out = subprocess.check_output(cmd, cwd=dest).decode('utf-8')
out = processutils.check_output(cmd, cwd=dest).decode('utf-8')
print('Results: %s' % (out,))
name = out.splitlines()[-1].strip()
return name

4
openstack_releases/requirements.py
View File

@ -17,11 +17,11 @@
import logging
import os.path
import subprocess
import pkg_resources
from openstack_releases import gitutils
from openstack_releases import processutils
from openstack_releases import pythonutils
from openstack_releases import versionutils
@ -89,7 +89,7 @@ def get_min_specifier(specifier_set):
def get_requirements_at_ref(workdir, repo, ref):
"Check out the repo at the ref and load the list of requirements."
dest = gitutils.clone_repo(workdir, repo, ref=ref)
subprocess.check_call(['python', 'setup.py', 'sdist'], cwd=dest)
processutils.check_call(['python', 'setup.py', 'sdist'], cwd=dest)
sdist_name = pythonutils.get_sdist_name(workdir, repo)
requirements_filename = os.path.join(
dest, sdist_name + '.egg-info', 'requires.txt',