releases/openstack_releases/gitutils.py
Doug Hellmann 34756ddbdf replace the use of zuul-cloner with a shell script
Under zuulv3 the behavior has changed significantly to now require
specifying which repositories need to be cloned for each job, and
zuul-cloner only copies the repositories that are already
available. As soon as zuulv3 is released, our jobs relying on the old
fallback behavior of checking out from a cache when present but using
the upstream server in other situations will fail.

The new script tools/clone_repo.sh handles the cases we have in the
tools and jobs within this repository, and should also be usable from
the tag-releases job.

zuul is removed from the list of requirements so it is no longer
installed when jobs run under tox.

list_unreleased_changes.sh is updated to drop the use of the
virtualenv, which was only present for zuul-cloner.

The other updates in the patch are to change the tools to use the new
script.

Change-Id: Ic559d27881a9a89fb0dcb5295a4d7ed7c578112f
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
2017-10-17 15:24:30 -04:00

339 lines
11 KiB
Python

# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
import os.path
import subprocess
from openstack_releases import links
# Disable warnings about insecure connections.
from requests.packages import urllib3
urllib3.disable_warnings()
LOG = logging.getLogger(__name__)
CGIT_SHA_TEMPLATE = 'http://git.openstack.org/cgit/%s/commit/?id=%s'
CGIT_TAG_TEMPLATE = 'http://git.openstack.org/cgit/%s/tag/?h=%s'
def find_modified_deliverable_files():
"Return a list of files modified by the most recent commit."
results = subprocess.check_output(
['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^']
).decode('utf-8')
filenames = [
l.strip()
for l in results.splitlines()
if l.startswith('deliverables/')
]
return filenames
def commit_exists(workdir, repo, ref):
"""Return boolean specifying whether the reference exists in the repository.
The commit must have been merged into the repository, but this
check does not enforce any branch membership.
"""
try:
subprocess.check_output(
['git', 'show', ref],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
except subprocess.CalledProcessError as err:
LOG.error('Could not find {}: {}'.format(ref, err))
return False
return True
def tag_exists(repo, ref):
"""Return boolean specifying whether the reference exists in the repository.
Uses a cgit query instead of looking locally to avoid cloning a
repository or having Depends-On settings in a commit message allow
someone to fool the check.
"""
url = CGIT_TAG_TEMPLATE % (repo, ref)
return links.link_exists(url)
def ensure_basic_git_config(workdir, repo, settings):
"""Given a repo directory and a settings dict, set local config values
if those settings are not already defined.
"""
dest = os.path.join(workdir, repo)
for key, value in settings.items():
LOG.info('looking for git config {}'.format(key))
try:
existing = subprocess.check_output(
['git', 'config', '--get', key],
cwd=dest,
).decode('utf-8').strip()
LOG.info('using existing setting of {}: {!r}'.format(key, existing))
except subprocess.CalledProcessError:
LOG.info('updating setting of {} to {!r}'.format(key, value))
subprocess.check_call(
['git', 'config', key, value],
cwd=dest,
)
def clone_repo(workdir, repo, ref=None, branch=None):
"Check out the code."
print('\nChecking out repository {} to {}'.format(
repo, branch or ref or 'master'))
cmd = [
'./tools/clone_repo.sh',
'--workspace', workdir,
]
if ref:
cmd.extend(['--ref', ref])
if branch:
cmd.extend(['--branch', branch])
cmd.append(repo)
subprocess.check_call(cmd)
def sha_for_tag(workdir, repo, version):
"""Return the SHA for a given tag
"""
# git log 2.3.11 -n 1 --pretty=format:%H
try:
actual_sha = subprocess.check_output(
['git', 'log', str(version), '-n', '1', '--pretty=format:%H'],
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8')
actual_sha = actual_sha.strip()
except subprocess.CalledProcessError as e:
LOG.info('ERROR getting SHA for tag %r: %s [%s]',
version, e, e.output.strip())
actual_sha = ''
return actual_sha
def _filter_branches(output):
"Strip garbage from branch list output"
return [
n
for n in output.strip().split()
if '/' in n or n == 'master'
]
def stable_branch_exists(workdir, repo, series):
"Does the stable/series branch exist?"
remote_match = 'remotes/origin/stable/%s' % series
try:
containing_branches = _filter_branches(
subprocess.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
)
return (remote_match in containing_branches)
except subprocess.CalledProcessError as e:
LOG.error('failed checking for branch: %s [%s]', e, e.output.strip())
return False
def check_branch_sha(workdir, repo, series, sha):
"""Check if the SHA is in the targeted branch.
The SHA must appear on a stable/$series branch (if it exists) or
master (if stable/$series does not exist). It is up to the
reviewer to verify that releases from master are in a sensible
location relative to other existing branches.
We do not compare $series against the existing branches ordering
because that would prevent us from retroactively creating a stable
branch for a project after a later stable branch is created (i.e.,
if stable/N exists we could not create stable/N-1).
"""
remote_match = 'remotes/origin/stable/%s' % series
try:
containing_branches = _filter_branches(
subprocess.check_output(
['git', 'branch', '-a', '--contains', sha],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
)
# If the patch is on the named branch, everything is fine.
if remote_match in containing_branches:
return True
# If the expected branch does not exist yet, this may be a
# late release attempt to create that branch or just a project
# that hasn't branched, yet, and is releasing from master for
# that series. Allow the release, as long as it is on the
# master branch.
all_branches = _filter_branches(
subprocess.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
)
if (remote_match not in all_branches) and ('master' in containing_branches):
return True
# At this point we know the release is not from the required
# branch and it is not from master, which means it is the
# wrong branch and should not be allowed.
return False
except subprocess.CalledProcessError as e:
LOG.error('failed checking SHA on branch: %s [%s]' % (e, e.output.strip()))
return False
def check_ancestry(workdir, repo, old_version, sha):
"Check if the SHA is in the ancestry of the previous version."
try:
ancestors = subprocess.check_output(
['git', 'log', '--oneline', '--ancestry-path',
'%s..%s' % (old_version, sha)],
cwd=os.path.join(workdir, repo),
).decode('utf-8').strip()
return bool(ancestors)
except subprocess.CalledProcessError as e:
LOG.error('failed checking ancestry: %s [%s]' % (e, e.output.strip()))
return False
def get_latest_tag(workdir, repo, sha=None):
cmd = ['git', 'describe', '--abbrev=0', '--always']
if sha is not None:
cmd.append(sha)
try:
return subprocess.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except subprocess.CalledProcessError as e:
LOG.warning('failed to retrieve latest tag: %s [%s]',
e, e.output.strip())
return None
def add_tag(workdir, repo, tag, sha):
cmd = ['git', 'tag', '-m', 'temporary tag', tag, sha]
try:
LOG.info(' '.join(cmd))
return subprocess.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except subprocess.CalledProcessError as e:
LOG.warning('failed to add tag: %s [%s]',
e, e.output.strip())
return None
def get_branches(workdir, repo):
try:
output = subprocess.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
# Example output:
# * (no branch)
# master
# stable/mitaka
# stable/newton
# stable/ocata
# remotes/origin/HEAD -> origin/master
# remotes/origin/master
# remotes/origin/stable/mitaka
# remotes/origin/stable/newton
# remotes/origin/stable/ocata
results = []
for line in output.splitlines():
branch = line.strip().lstrip('*').strip()
if branch.startswith('('):
continue
if '->' in branch:
continue
results.append(branch)
return results
except subprocess.CalledProcessError as e:
LOG.error('failed to retrieve list of branches: %s [%s]',
e, e.output.strip())
return []
def branches_containing(workdir, repo, ref):
try:
output = subprocess.check_output(
['git', 'branch', '-r', '--contains', ref],
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
# Example output:
# origin/stable/ocata
results = []
for line in output.splitlines():
results.append(line.strip())
return results
except subprocess.CalledProcessError as e:
LOG.error('failed to retrieve list of branches containing %s: %s [%s]',
ref, e, e.output.strip())
return []
def get_branch_base(workdir, repo, branch):
"Return SHA at base of branch."
# http://stackoverflow.com/questions/1527234/finding-a-branch-point-with-git
# git rev-list $(git rev-list --first-parent ^origin/stable/newton master | tail -n1)^^!
#
# Determine the first parent.
cmd = [
'git',
'rev-list',
'--first-parent',
'^origin/{}'.format(branch),
'master',
]
try:
parents = subprocess.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except subprocess.CalledProcessError as e:
LOG.warning('failed to retrieve branch base: %s [%s]',
e, e.output.strip())
return None
parent = parents.splitlines()[-1]
# Now get the ^^! commit
cmd = [
'git',
'rev-list',
'{}^^!'.format(parent),
]
try:
return subprocess.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except subprocess.CalledProcessError as e:
LOG.warning('failed to retrieve branch base: %s [%s]',
e, e.output.strip())
return None