openstack/releases
Code Issues Proposed changes

teach missing-releases to look for artifacts as well as tags

Browse Source 
We need a tool to verify that the release artifacts exist on the
tarballs site. The missing-releases command was already looking for
missing tags, so have it look for the tarballs, too.

This change refactors the URL generation code into a separate module so
it can be used by missing-releases and the sphinx extension. It also
creates a reusable function for testing a link, and has the gitutils
module functions responsible for looking for tags, branches, etc. use
it.

Change the default behavior to only look at the most recent release for
each deliverable, and add an option to go back to the previous behavior
of scanning all of them.

Change-Id: Ic7345466ccd83cf2d8d9d6d019107d6fbba171cc
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2016-12-15 17:59:21 -05:00
parent bd25182d1f
commit 4305adb190
4 changed files with 136 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
openstack_releases/cmds/missing.py
View File

@ -30,6 +30,7 @@ from requests.packages import urllib3
from openstack_releases import defaults
from openstack_releases import gitutils
from openstack_releases import links
urllib3.disable_warnings()
@ -40,6 +41,18 @@ def main():
'--series', '-s',
help='release series to scan',
)
parser.add_argument(
'--artifacts',
default=False,
action='store_true',
help='only scan the build artifacts',
)
parser.add_argument(
'--all',
default=False,
action='store_true',
help='scan all releases, not just most recent',
)
parser.add_argument(
'input',
nargs='*',
@ -51,9 +64,9 @@ def main():
if args.input:
filenames = args.input
elif args.series:
filenames = glob.glob('deliverables/%s/*.yaml' % args.series)
filenames = sorted(glob.glob('deliverables/%s/*.yaml' % args.series))
else:
filenames = gitutils.find_modified_deliverable_files()
filenames = sorted(gitutils.find_modified_deliverable_files())
if not filenames:
print('no modified deliverable files, validating all releases from %s'
% defaults.RELEASE)
@ -69,7 +82,13 @@ def main():
with open(filename, 'r') as f:
deliverable_info = yaml.load(f.read())
for release in deliverable_info['releases']:
link_mode = deliverable_info.get('artifact-link-mode', 'tarball')
releases = deliverable_info.get('releases', [])
if not args.all:
releases = releases[-1:]
for release in releases:
for project in release['projects']:
# Report if the version has already been
@ -78,19 +97,36 @@ def main():
# import history and sometimes we want to make new
# releases.
print('%s %s' % (project['repo'], release['version']), end=' ')
version_exists = gitutils.tag_exists(
project['repo'], release['version'],
)
if version_exists:
print('found')
else:
print('MISSING')
errors.append(
'%s missing tag %s' % (
project['repo'],
release['version'],
)
if not args.artifacts:
version_exists = gitutils.tag_exists(
project['repo'], release['version'],
)
if version_exists:
print('tag:found', end=' ')
else:
print('tag:MISSING', end=' ')
errors.append('%s missing tag %s' %
(project['repo'], release['version']))
# Look for the tarball associated with the tag and
# report if that exists.
if link_mode == 'tarball':
tb_url = links.tarball_url(release['version'], project)
if links.link_exists(tb_url):
print('tarball:found', end=' ')
else:
print('tarball:MISSING\n%s' % tb_url)
errors.append('%s missing tarball %s' %
(filename, tb_url))
sig_url = links.signature_url(release['version'], project)
if links.link_exists(sig_url):
print('signature:found', end=' ')
else:
print('signature:MISSING\n%s' % sig_url)
errors.append('%s missing signature %s' %
(filename, sig_url))
print()
if errors:
print('\n\n%s errors found' % len(errors))

15
openstack_releases/gitutils.py
View File

@ -16,12 +16,13 @@ import os
import os.path
import subprocess
import requests
from openstack_releases import links
# Disable warnings about insecure connections.
from requests.packages import urllib3
urllib3.disable_warnings()
CGIT_SHA_TEMPLATE = 'http://git.openstack.org/cgit/%s/commit/?id=%s'
CGIT_TAG_TEMPLATE = 'http://git.openstack.org/cgit/%s/tag/?h=%s'
@ -48,11 +49,7 @@ def commit_exists(repo, ref):
"""
url = CGIT_SHA_TEMPLATE % (repo, ref)
response = requests.get(url)
missing_commit = (
(response.status_code // 100 != 2) or 'Bad object id' in response.text
)
return not missing_commit
return links.link_exists(url)
def tag_exists(repo, ref):
@ -64,11 +61,7 @@ def tag_exists(repo, ref):
"""
url = CGIT_TAG_TEMPLATE % (repo, ref)
response = requests.get(url)
missing_commit = (
(response.status_code // 100 != 2) or 'Bad object id' in response.text
)
return not missing_commit
return links.link_exists(url)
def clone_repo(workdir, repo):

73
openstack_releases/links.py Normal file
View File

@ -0,0 +1,73 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import requests
def link_exists(url):
try:
response = requests.head(
url,
headers={'user-agent': 'openstack-release-link-checker'},
)
missing = (
(response.status_code // 100 != 2) or
'Bad object id' in response.text
)
except requests.exceptions.ConnectionError as e:
print('Failed to access %s: %s' % (url, e))
missing = True
return not missing
def tarball_url(version, project):
repo_base = project['repo'].rsplit('/')[-1]
base = project.get('tarball-base', repo_base)
return '{s}/{r}/{n}-{v}.tar.gz'.format(
s='https://tarballs.openstack.org',
v=version,
r=repo_base,
n=base,
)
def artifact_link(version, project, deliverable_info):
mode = deliverable_info.get('artifact-link-mode', 'tarball')
if mode == 'tarball':
# Link the version number to the tarball for downloading.
url = tarball_url(version, project)
return '`{v} <{url}>`__'.format(v=version, url=url)
elif mode == 'none':
# Only show the version number.
return version
raise ValueError('Unrecognized artifact-link-mode: %r' % mode)
def signature_url(version, project):
tb_url = tarball_url(version, project)
return tb_url + '.asc'
def artifact_signature_link(version, type, project, deliverable_info):
mode = deliverable_info.get('artifact-link-mode', 'tarball')
if mode == 'tarball':
url = signature_url(version, project)
# Link the signature type to the tarball for downloading.
return '`{t} <{url}>`__'.format(
t=type,
url=url,
)
elif mode == 'none':
return ""
raise ValueError('Unrecognized artifact-link-mode: %r' % mode)

53
openstack_releases/sphinxext.py
View File

@ -23,6 +23,7 @@ from docutils.statemachine import ViewList
from sphinx.util.nodes import nested_parse_with_titles
from openstack_releases import deliverable
from openstack_releases import links
def _list_table(add, headers, data, title='', columns=None):
@ -174,46 +175,6 @@ class DeliverableDirectiveBase(rst.Directive):
'cycle-trailing': 'Projects Trailing the Release Cycle',
}
@staticmethod
def _artifact_link(mode, version, project):
if mode == 'tarball':
# Link the version number to the tarball for downloading.
repo_base = project['repo'].rsplit('/')[-1]
if 'tarball-base' in project:
base = project['tarball-base']
else:
base = repo_base
return '`{v} <{s}/{r}/{n}-{v}.tar.gz>`__'.format(
s='https://tarballs.openstack.org',
v=version,
r=repo_base,
n=base,
)
elif mode == 'none':
# Only show the version number.
return version
raise ValueError('Unrecognized artifact-link-mode: %r' % mode)
@staticmethod
def _artifact_signature_link(mode, version, type, project):
if mode == 'tarball':
# Link the version number to the tarball for downloading.
repo_base = project['repo'].rsplit('/')[-1]
if 'tarball-base' in project:
base = project['tarball-base']
else:
base = repo_base
return '`{t} <{s}/{r}/{n}-{v}.tar.gz.asc>`__'.format(
s='https://tarballs.openstack.org',
v=version,
t=type,
r=repo_base,
n=base,
)
elif mode == 'none':
return ""
raise ValueError('Unrecognized artifact-link-mode: %r' % mode)
def _add_deliverables(self, type_tag, deliverables, series, app, result):
source_name = '<' + __name__ + '>'
@ -306,20 +267,22 @@ class DeliverableDirectiveBase(rst.Directive):
_add('')
_add('Release Notes: %s' % notes_link)
_add('')
link_mode = deliverable_info.get('artifact-link-mode', 'tarball')
# We have signatures for artifacts only after newton
if series and series[0] >= 'o':
headers = ['Version', 'Signature', 'Repo', 'Git Commit']
data = ((self._artifact_link(link_mode, r['version'], p),
self._artifact_signature_link(link_mode, r['version'],
'pgp', p),
data = ((links.artifact_link(r['version'], p,
deliverable_info),
links.artifact_signature_link(r['version'],
'pgp', p,
deliverable_info),
p['repo'], p['hash'])
for r in reversed(deliverable_info.get('releases', []))
for p in r.get('projects', []))
columns = [10, 10, 40, 50]
else:
headers = ['Version', 'Repo', 'Git Commit']
data = ((self._artifact_link(link_mode, r['version'], p),
data = ((links.artifact_link(r['version'], p,
deliverable_info),
p['repo'], p['hash'])
for r in reversed(deliverable_info.get('releases', []))
for p in r.get('projects', []))