releases/openstack_releases/cmds/missing.py
Sean McGinnis d403ccf5fa
Update missing cmd to follow redirects
Handles redirection in the link checking for artifacts.

Also drops looking for py2 wheels since we now only have py3.

Change-Id: I34b201902faa60795dae44ebf059af0f2f9242c6
Signed-off-by: Sean McGinnis <sean.mcginnis@gmail.com>
2020-05-13 07:19:35 -05:00

210 lines
7.6 KiB
Python

# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Look for releases listed in a series but not actually tagged."""
import argparse
import glob
import os
import os.path
# Disable warnings about insecure connections.
from requests.packages import urllib3
from openstack_releases import defaults
from openstack_releases import deliverable
from openstack_releases import gitutils
from openstack_releases import links
from openstack_releases import pythonutils
urllib3.disable_warnings()
def check_url(type, url):
if links.link_exists(url):
print(' found {}'.format(type))
else:
print(' did not find {} {}'.format(type, url))
yield 'missing {} {}'.format(type, url)
def check_signed_file(type, url):
for item_type, item in [(type, url), (type + ' signature', url + '.asc')]:
yield from check_url(item_type, item)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--series', '-s',
help='release series to scan',
)
parser.add_argument(
'--artifacts',
default=False,
action='store_true',
help='only scan the build artifacts',
)
parser.add_argument(
'--all',
default=False,
action='store_true',
help='scan all releases, not just most recent',
)
parser.add_argument(
'input',
nargs='*',
help=('YAML files to validate, defaults to '
'files changed in the latest commit'),
)
args = parser.parse_args()
if args.input:
filenames = args.input
elif args.series:
filenames = sorted(glob.glob('deliverables/%s/*.yaml' % args.series))
else:
filenames = sorted(gitutils.find_modified_deliverable_files())
if not filenames:
print('no modified deliverable files, validating all releases from %s'
% defaults.RELEASE)
filenames = glob.glob('deliverables/' + defaults.RELEASE + '/*.yaml')
errors = []
for filename in filenames:
# Skip our test deliverable
if 'release-test' in filename:
continue
print('\nChecking %s' % filename)
if not os.path.exists(filename):
print("File was deleted, skipping.")
continue
deliv = deliverable.Deliverable.read_file(filename)
releases = deliv.releases
if not args.all:
releases = releases[-1:]
for release in releases:
version = release.version
for project in release.projects:
# Report if the version has already been
# tagged. We expect it to not exist, but neither
# case is an error because sometimes we want to
# import history and sometimes we want to make new
# releases.
print('\n%s %s' % (project.repo.name, version))
if not args.artifacts:
version_exists = gitutils.tag_exists(
project.repo.name, version,
)
if version_exists:
print(' found tag')
else:
print(' did not find tag')
errors.append('%s missing tag %s' %
(project.repo.name, version))
# Look for the tarball associated with the tag and
# report if that exists.
if deliv.artifact_link_mode == 'tarball':
tb_url = links.tarball_url(version, project)
errors.extend(check_signed_file('tarball', tb_url))
if 'a' in version or 'b' in version or 'rc' in version:
print(' pre-releases are not uploaded to PyPI')
continue
pypi_name = project.repo.pypi_name
if not pypi_name:
pypi_name = project.guess_sdist_name()
pypi_info = pythonutils.get_pypi_info(pypi_name)
if not pypi_info:
print(' apparently not a python module')
continue
wheel_errors = list(
check_url(
'python 3 wheel',
links.wheel_py3_url(version, project)
)
)
has_23_wheel = False
if wheel_errors:
has_23_wheel = True
# Check if there is actually a 2/3 wheel
wheel_errors = list(
check_url(
'python 2/3 wheel',
links.wheel_both_url(version, project)
)
)
if wheel_errors:
# We are missing the wheel.
errors.extend(wheel_errors)
elif has_23_wheel:
# We have the "both" wheel, so check for the
# signature file.
errors.extend(
check_url(
'python 2/3 wheel signature',
links.wheel_both_url(version,
project) + '.asc',
)
)
else:
# We have the py3 wheel, so check for the
# signature file.
errors.extend(
check_url(
'python 3 wheel signature',
links.wheel_py3_url(version,
project) + '.asc',
)
)
if version not in pypi_info.get('releases', {}):
msg = ('{} dist with version {} '
'not uploaded to PyPI').format(
pypi_name, version)
print(' {}'.format(msg))
errors.append(msg)
else:
print(' found version {} on PyPI'.format(
version))
expected_types = set(['bdist_wheel', 'sdist'])
actual_types = set(
r['packagetype']
for r in pypi_info['releases'][version]
)
for actual in actual_types:
print(' found {} on PyPI'.format(actual))
for missing in expected_types.difference(actual_types):
msg = '{} not found on PyPI'.format(missing)
print(' {}'.format(msg))
errors.append(msg)
if errors:
print('\n\n%s errors found' % len(errors))
for e in errors:
print(e)
return 1 if errors else 0