# All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """Try to verify that the latest commit contains valid SHA values. """ from __future__ import print_function import argparse import atexit import collections import functools import glob import inspect import logging import os import os.path import re import shutil import sys import tempfile from openstack_governance import governance import requests import six # Disable warnings about insecure connections. from requests.packages import urllib3 from openstack_releases import defaults from openstack_releases import deliverable from openstack_releases import gitutils from openstack_releases import npmutils from openstack_releases import project_config from openstack_releases import puppetutils from openstack_releases import pythonutils from openstack_releases import requirements from openstack_releases import versionutils from openstack_releases import xstaticutils LOG = logging.getLogger() urllib3.disable_warnings() _CLOSED_SERIES = set([ 'austin', 'bexar', 'cactus', 'diablo', 'essex', 'folsom', 'grizzly', 'havana', 'icehouse', 'juno', 'kilo', 'liberty', 'mitaka', ]) _USES_PREVER = set([ 'cycle-with-milestones', 'cycle-trailing', 'cycle-with-rc', ]) _VALID_BRANCH_PREFIXES = set([ 'stable', 'feature', ]) _NO_STABLE_BRANCH_CHECK = set([ 'gnocchi', 'rally', 'puppet-pacemaker', # tracks upstream version ]) _TYPE_TO_RELEASE_TYPE = { 'library': 'python-pypi', 'service': 'python-service', 'horizon-plugin': 'horizon', } _PYTHON_RELEASE_TYPES = ['python-service', 'python-pypi', 'neutron', 'horizon'] _PLEASE = ('It is too expensive to determine this value during ' 'the site build, please set it explicitly.') def header(title, underline='-'): print('\n%s' % title) print(underline * len(title)) def is_a_hash(val): "Return bool indicating if val looks like a valid hash." return re.search('^[a-f0-9]{40}$', val, re.I) is not None def applies_to_current(f): @functools.wraps(f) def decorated(deliv, context): if deliv.series != defaults.RELEASE: print('this rule only applies to the most current series, skipping') return return f(deliv, context) return decorated def applies_to_released(f): @functools.wraps(f) def decorated(deliv, context): if not deliv.is_released: print('no releases, skipping') return return f(deliv, context) return decorated def applies_to_cycle(f): @functools.wraps(f) def decorated(deliv, context): if deliv.is_independent: print('rule does not apply to independent projects') return return f(deliv, context) return decorated # Remember which tags already exist so we don't have to repeat the # expensive check. existing_tag_cache = collections.defaultdict(set) def includes_new_tag(deliv, context): "Return true if the deliverable is describing a new tag." for release in deliv.releases: for project in release.projects: if project.repo.is_retired: LOG.info('{} is retired, skipping'.format(project.repo.name)) continue if release.version in existing_tag_cache[project.repo.name]: LOG.debug('%s already tagged %s, skipping', project.repo.name, release.version) continue version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) if version_exists: existing_tag_cache[project.repo.name].add(release.version) LOG.debug('%s already tagged %s, skipping', project.repo.name, release.version) else: return True return False def skip_existing_tags(f): @functools.wraps(f) def decorated(deliv, context): if includes_new_tag(deliv, context): return f(deliv, context) else: print('This rule only applies to new tags.') return decorated @skip_existing_tags @applies_to_cycle @applies_to_released @applies_to_current def validate_series_open(deliv, context): "No releases in the new series until the previous one has a branch." deliverables_dir = os.path.dirname( os.path.dirname(context.filename) ) deliverable_base = os.path.basename(context.filename) pattern = os.path.join( deliverables_dir, '*', deliverable_base, ) # NOTE(dhellmann): When projects switch from _independent to # cycle-based models, we don't want to require a # stable/_independent branch, so ignore those files. all_deliverable_files = [ name for name in sorted(glob.glob(pattern)) if '/_independent/' not in name ] idx = all_deliverable_files.index(context.filename) if idx == 0: # This is the first cycle-based deliverable file. print('this is the first cycle-based version of this deliverable, ' 'skipping further checks') return previous_deliverable_file = all_deliverable_files[idx - 1] previous_series = os.path.basename( os.path.dirname(previous_deliverable_file) ) expected_branch = 'stable/' + previous_series previous_deliverable = deliverable.Deliverable.read_file( previous_deliverable_file ) for branch in previous_deliverable.branches: if branch.name == expected_branch: # Everything is OK print('found branch {} in {}'.format( branch.name, previous_deliverable_file, )) return context.warning( 'There is no {} branch defined in {}. Is the {} series open?'.format( expected_branch, previous_deliverable_file, deliv.series)) @skip_existing_tags @applies_to_released @applies_to_cycle def validate_series_first(deliv, context): "The first release in a series needs to end with '.0'." releases = deliv.releases if len(releases) != 1: # We only have to check this when the first release is being # applied in the file. print('this rule only applies to the first release in a series') return versionstr = releases[0].version patchlevel = versionstr.rpartition('.')[-1] if not (patchlevel == '0' or patchlevel.startswith('0b')): context.error( 'Initial releases in a series must increment at ' 'least the minor version or be beta versions. %r' % (versionstr,) ) @skip_existing_tags @applies_to_current @applies_to_released @applies_to_cycle def validate_pre_release_progression(deliv, context): "Pre-release versions must be applied in progressive order" if not deliv.is_milestone_based: print('this rule only applies to milestone-based projects') return releases = deliv.releases if len(releases) < 2: print('this rule only applies to the final release in a series') return previous_release = releases[-2] current_release = releases[-1] LOG.debug( 'checking progression from {} to {}'.format( previous_release.version, current_release.version) ) if 'rc' in current_release.version: version_type = 'Release candidate' previous_type = 'a beta or release candidate' allowed = ['b', 'rc'] elif 'b' in current_release.version: version_type = 'Beta' previous_type = 'an alpha or beta' allowed = ['a', 'b'] elif 'a' in current_release.version: version_type = 'Alpha' previous_type = 'an alpha' allowed = ['a'] else: # Final versions must come after release candidates # or other final versions. version_type = 'Final' previous_type = 'a release candidate or final' allowed = ['rc', 'final'] def checks(): for pre in allowed: if pre == 'final': yield not previous_release.is_pre_release_version else: yield pre in previous_release.version if not any(checks()): context.error( ('{} version {} must come after ' '{} version, not {}').format( version_type, current_release.version, previous_type, previous_release.version) ) else: print('OK') @skip_existing_tags @applies_to_released def validate_series_final(deliv, context): "The final release after a RC should tag the same commit." releases = deliv.releases if len(releases) < 2: # We only have to check this when the first release is being # applied in the file. print('this rule only applies to the final release in a series') return previous_release = releases[-2] current_release = releases[-1] if (current_release.is_release_candidate or not previous_release.is_release_candidate): print('this rule only applies when tagging a final from a candidate') return current_projects = sorted(releases[-1].projects) previous_projects = sorted(previous_release.projects) for c_proj, p_proj in zip(current_projects, previous_projects): LOG.debug( 'comparing {}:{} with {}:{}'.format( c_proj.repo.name, c_proj.hash, p_proj.repo.name, p_proj.hash, )) if c_proj.repo.name != p_proj.repo.name: context.error( '{} does not match {} so there is some missing info ' 'in this release'.format(c_proj.repo.name, p_proj.repo.name) ) elif c_proj.hash != p_proj.hash: context.error( '{} for {} is on {} but should be {} ' 'to match version {}'.format( current_release.version, c_proj.repo.name, c_proj.hash, p_proj.hash, previous_release.version) ) else: print('OK') def _require_tag_on_all_repos(deliv, current_release, eol_or_em, context): # The tag should be applied to all of the repositories for the # deliverable. actual_repos = set(p.repo.name for p in current_release.projects) expected_repos = set(r.name for r in deliv.repos) error = False for extra in actual_repos.difference(expected_repos): error = True context.error( '%s release %s includes repository %s ' 'that is not in deliverable' % (eol_or_em, current_release.version, extra) ) for missing in expected_repos.difference(actual_repos): error = True context.error( 'release %s is missing %s, ' 'which appears in the deliverable' % (current_release.version, missing) ) if not error: print('OK') @skip_existing_tags @applies_to_released def validate_series_eol(deliv, context): "The EOL tag should be applied to all repositories." current_release = deliv.releases[-1] if not current_release.is_eol: print('this rule only applies when tagging a series as end-of-life') return _require_tag_on_all_repos( deliv, current_release, 'EOL', context, ) @skip_existing_tags @applies_to_released def validate_series_em(deliv, context): "The EM tag should be applied to the previous release." current_release = deliv.releases[-1] if not current_release.is_em: print('this rule only applies when tagging ' 'a series as extended-maintenance') return _require_tag_on_all_repos( deliv, current_release, 'extended maintenance', context, ) def validate_bugtracker(deliv, context): "Does the bug tracker info link to something that exists?" lp_name = deliv.launchpad_id sb_id = deliv.storyboard_id if lp_name: try: lp_resp = requests.get('https://api.launchpad.net/1.0/' + lp_name) except requests.exceptions.ConnectionError as e: # The flakey Launchpad API failed. Don't punish the user for that. context.warning('Could not verify launchpad project %s (%s)' % (lp_name, e)) else: if (lp_resp.status_code // 100) == 4: context.error('Launchpad project %s does not exist' % lp_name) print('launchpad project ID {} OK'.format(lp_name)) elif sb_id: try: projects_resp = requests.get( 'https://storyboard.openstack.org/api/v1/projects' ) except requests.exceptions.ConnectionError as e: # The flakey Launchpad API failed. Don't punish the user for that. context.warning('Could not verify storyboard project %s (%s)' % (sb_id, e)) else: if (projects_resp.status_code // 100) == 4: context.warning( 'Could not verify storyboard project, API call failed.' ) for project in projects_resp.json(): # TODO(fungi): This can be changed to simply check # that sb_id == project.get('name') later if all the # data gets updated from numbers to names. if sb_id in (project.get('id'), project.get('name')): break else: context.error( 'Did not find a storyboard project with ID %s' % sb_id ) print('storyboard project ID {} OK'.format(sb_id)) else: context.error('No launchpad or storyboard project given') def validate_team(deliv, context): "Look for the team name in the governance data." try: context.gov_data.get_team(deliv.team) except ValueError: context.warning( 'Team {} not in governance data. ' 'Only official teams should use this repository ' 'for managing releases. See README.rst for details.'.format( deliv.team)) else: print('owned by team {}'.format(deliv.team)) def validate_release_notes(deliv, context): "Make sure the release notes page exists, if it is specified." notes_link = deliv.release_notes if not notes_link: print('no release-notes given, skipping') return if isinstance(notes_link, dict): # Dictionary mapping repositories to links. We don't want any # repositories that are not known, so check that as well as # the actual links. for repo_name in sorted(notes_link.keys()): if repo_name not in deliv.known_repo_names: context.error( 'linking to release notes for unknown ' 'repository {}'.format( repo_name) ) links = list(notes_link.values()) else: links = [notes_link] for link in links: rn_resp = requests.get(link) if (rn_resp.status_code // 100) != 2: context.error('Could not fetch release notes page %s: %s' % (link, rn_resp.status_code)) else: print('{} OK'.format(link)) def validate_model(deliv, context): "Require a valid release model" LOG.debug('release model {}'.format(deliv.model)) if not deliv.is_independent and not deliv.model: # If the deliverable is not independent it must declare a # release model. context.error( 'no release-model specified', ) if deliv.model == 'independent' and deliv.series != 'independent': # If the project is release:independent, make sure # that's where the deliverable file is. context.error( 'uses the independent release model ' 'and should be in the _independent ' 'directory' ) # If the project is declaring some other release model, make # sure it is not in the _independent directory. We have to # bypass the model property because that always returns # 'independent' for deliverables in that series. model_value = deliv.data.get('release-model', 'independent') if deliv.series == 'independent' and model_value != 'independent': context.error( 'deliverables in the _independent directory ' 'should all use the independent release model' ) if deliv.model == 'untagged' and deliv.is_released: context.error( 'untagged deliverables should not have a "releases" section' ) return def clone_deliverable(deliv, context): """Clone all of the repositories for the deliverable into the workdir. Returns boolean indicating whether all of the clones could be performed as expected. """ cloned = set() ok = True for repo in deliv.repos: if repo.name in cloned: continue if repo.is_retired: LOG.info('{} is retired, skipping clone'.format(repo.name)) continue if not gitutils.safe_clone_repo(context.workdir, repo.name, 'master', context): ok = False return ok def _require_gitreview(repo, context): LOG.debug('looking for .gitreview in %s' % repo) filename = os.path.join( context.workdir, repo, '.gitreview', ) if not os.path.exists(filename): context.error('%s has no .gitreview file' % (repo,)) else: print('found {}'.format(filename)) @skip_existing_tags def validate_gitreview(deliv, context): "All repos must include a .gitreview file for new releases." checked = set() for release in deliv.releases: for project in release.projects: if project.repo.name in checked: continue checked.add(project.repo.name) if project.repo.is_retired: print('{} is retired, skipping'.format( project.repo.name)) continue version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) if not version_exists: LOG.debug('checking {} at {} for {}'.format( project.repo.name, project.hash, release.version)) gitutils.checkout_ref( context.workdir, project.repo.name, project.hash, context) _require_gitreview(project.repo.name, context) else: print('version {} exists, skipping'.format( release.version)) def get_release_type(deliv, repo, workdir): "Return tuple with release type and whether it was explicitly set." if deliv.release_type is not None: return (deliv.release_type, True) from_type = _TYPE_TO_RELEASE_TYPE.get(deliv.type) if from_type is not None: return (from_type, False) if deliv.include_pypi_link: return ('python-pypi', False) if puppetutils.looks_like_a_module(workdir, repo.name): return ('puppet', False) if npmutils.looks_like_a_module(workdir, repo.name): return ('nodejs', False) return ('python-service', False) @skip_existing_tags @applies_to_released def validate_release_type(deliv, context): "Does the most recent release comply with the rules for the release-type?" if deliv.artifact_link_mode == 'none': print('link-mode is "none", skipping release-type checks') return release = deliv.releases[-1] for project in release.projects: LOG.debug('checking release-type for {}'.format(project.repo.name)) release_type, was_explicit = get_release_type( deliv, project.repo, context.workdir, ) if was_explicit: LOG.debug('found explicit release-type {!r}'.format( release_type)) else: LOG.debug('release-type not given, ' 'guessing {!r}'.format(release_type)) version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) if not version_exists: LOG.debug('new version {}, checking release jobs'.format( release.version)) gitutils.checkout_ref( context.workdir, project.repo.name, project.hash, context) project_config.require_release_jobs_for_repo( deliv, project.repo, release_type, context, ) @applies_to_released def validate_tarball_base(deliv, context): "Does tarball-base match the expected value?" if deliv.artifact_link_mode != 'tarball': print('rule does not apply for link-mode {}, skipping'.format( deliv.artifact_link_mode)) return release = deliv.releases[-1] for project in release.projects: version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) # Check that the sdist name and tarball-base name match. try: sdist = pythonutils.get_sdist_name(context.workdir, project.repo.name) except Exception as err: msg = 'Could not get the name of {} for version {}: {}'.format( project.repo.name, release.version, err) if version_exists: # If there was a problem with an existing # release, treat it as a warning so we # don't prevent new releases. context.warning(msg) else: context.error(msg) else: if sdist is not None: tarball_base = project.tarball_base expected = tarball_base or os.path.basename(project.repo.name) if sdist != expected: if tarball_base: action = 'is set to' else: action = 'defaults to' context.error( ('tarball-base for %s %s %s %r ' 'but the sdist name is actually %r. ' + _PLEASE) % (project.repo.name, release.version, action, expected, sdist)) else: print('{!r} matches expected {!r}'.format( sdist, expected)) @applies_to_released def validate_build_sdist(deliv, context): "Can we build an sdist for a python project?" release = deliv.releases[-1] for project in release.projects: version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) if version_exists: print('version {} was already tagged, skipping'.format( release.version)) continue gitutils.checkout_ref( context.workdir, project.repo.name, project.hash, context) try: pythonutils.build_sdist( context.workdir, project.repo.name) except Exception as err: context.error( 'Failed to build sdist for {}: {}'.format( project.repo.name, err)) @skip_existing_tags @applies_to_released def validate_pypi_readme(deliv, context): "Does the README look right for PyPI?" # Check out the repositories to the hash for the latest release on # the branch so we can find the setup.py and README.rst there (if # it exists), in case that has been removed from master after a # project is retired. This also ensures we get the right name for # the branch, in case the sdist name changes over time. latest_release = deliv.releases[-1] for project in latest_release.projects: gitutils.checkout_ref( context.workdir, project.repo.name, project.hash, context) if latest_release.is_eol: print('skipping README validation for EOL tag {}'.format( latest_release.version)) return if latest_release.is_em: print('skipping README validation for EM tag {}'.format( latest_release.version)) return for repo in deliv.repos: job_templates = context.zuul_projects.get(repo.name, {}).get( 'templates', []) LOG.debug('{} has job templates {}'.format(repo.name, job_templates)) # Look for jobs that appear to be talking about publishing to # PyPI. There are variations. pypi_jobs = [ j for j in job_templates if 'pypi' in j ] if not pypi_jobs: print('rule only applies to repos publishing to PyPI') continue LOG.debug('{} publishes to PyPI via {}'.format(repo.name, pypi_jobs)) try: pythonutils.check_readme_format(context.workdir, repo.name) except Exception as err: context.error('README check for {} failed: {}'.format( repo.name, err)) else: print('OK') @skip_existing_tags @applies_to_released def validate_pypi_permissions(deliv, context): "Do we have permission to upload to PyPI?" # Check out the repositories to the hash for the latest release on # the branch so we can find the setup.py there (if it exists), in # case that has been removed from master after a project is # retired. This also ensures we get the right name for the branch, # in case the sdist name changes over time. latest_release = deliv.releases[-1] for project in latest_release.projects: gitutils.checkout_ref( context.workdir, project.repo.name, project.hash, context) for repo in deliv.repos: job_templates = context.zuul_projects.get(repo.name, {}).get( 'templates', []) LOG.debug('{} has job templates {}'.format(repo.name, job_templates)) # Look for jobs that appear to be talking about publishing to # PyPI. There are variations. pypi_jobs = [ j for j in job_templates if 'pypi' in j ] if not pypi_jobs: print('rule does not apply to repos not publishing to PyPI') continue LOG.debug('{} publishes to PyPI via {}'.format(repo.name, pypi_jobs)) pypi_name = repo.pypi_name if not pypi_name: try: sdist = pythonutils.get_sdist_name(context.workdir, repo.name) except Exception as err: context.warning( 'Could not determine the sdist name ' 'for {} to check PyPI permissions: {}'.format( repo.name, err) ) return LOG.debug('using sdist name as pypi-name {!r}'.format(sdist)) pypi_name = sdist if not pypi_name: context.error( 'Could not determine a valid PyPI dist name for {}'.format( deliv.name)) return uploaders = pythonutils.get_pypi_uploaders(pypi_name) if not uploaders: context.error( 'could not find users with permission to upload packages ' 'for {}. Is the sdist name correct?'.format(pypi_name) ) elif 'openstackci' not in uploaders: context.error( 'openstackci does not have permission to upload packages ' 'for {}. Current owners include: {}'.format( pypi_name, ', '.join(sorted(uploaders))) ) else: print('found {} able to upload to {}'.format( sorted(uploaders), pypi_name)) @skip_existing_tags @applies_to_released def validate_release_sha_exists(deliv, context): "Ensure the hashes for each release exist." for release in deliv.releases: LOG.debug('checking {}'.format(release.version)) for project in release.projects: # Check the SHA specified for the tag. LOG.debug('{} SHA {}'.format(project.repo.name, project.hash)) if not is_a_hash(project.hash): context.error( ('%(repo)s version %(version)s release from ' '%(hash)r, which is not a hash') % { 'repo': project.repo.name, 'hash': project.hash, 'version': release.version} ) continue if not gitutils.checkout_ref(context.workdir, project.repo.name, project.hash, context): continue print('successfully cloned {}'.format(project.hash)) # Report if the SHA exists or not (an error if it # does not). sha_exists = gitutils.commit_exists( context.workdir, project.repo.name, project.hash, ) if not sha_exists: context.error('No commit %(hash)r in %(repo)r' % {'hash': project.hash, 'repo': project.repo.name}) @applies_to_released def validate_existing_tags(deliv, context): "Ensure tags that exist point to the SHAs listed." for release in deliv.releases: LOG.debug('checking {}'.format(release.version)) for project in release.projects: LOG.debug('{} SHA {}'.format(project.repo.name, project.hash)) if not gitutils.checkout_ref(context.workdir, project.repo.name, project.hash, context): continue # Report if the version has already been # tagged. We expect it to not exist, but neither # case is an error because sometimes we want to # import history and sometimes we want to make new # releases. version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) if not version_exists: print('{} does not have {} tag yet, skipping'.format( project.repo.name, release.version)) continue actual_sha = gitutils.sha_for_tag( context.workdir, project.repo.name, release.version, ) if actual_sha != project.hash: context.error( 'Version {} in {} is on ' 'commit {!r} instead of {!r}'.format( release.version, project.repo.name, actual_sha, project.hash) ) else: print('{} tag exists and is correct for {}'.format( release.version, project.repo.name)) @skip_existing_tags @applies_to_released def validate_version_numbers(deliv, context): "Ensure the version numbers are valid." # Track the previous version tag attached to each repository, by # name. prev_version = {} for release in deliv.releases: LOG.debug('checking {}'.format(release.version)) if release.is_eol: LOG.debug('Found new EOL tag {} for {}'.format( release.version, deliv.name)) if release.eol_series != deliv.series: context.error( 'EOL tag {} does not refer to the {} series.'.format( release.version, deliv.series)) continue if release.is_em: LOG.debug('Found new EM tag {} for {}'.format( release.version, deliv.name)) if release.em_series != deliv.series: context.error( 'EM tag {} does not refer to the {} series.'.format( release.version, deliv.series)) continue for project in release.projects: if not gitutils.checkout_ref(context.workdir, project.repo.name, project.hash, context): continue version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) if version_exists: print('tag exists, skipping further validation') continue LOG.debug('Found new version {} for {}'.format( release.version, project.repo)) release_type, was_explicit = get_release_type( deliv, project.repo, context.workdir, ) if was_explicit: LOG.debug('found explicit release-type {!r}'.format( release_type)) else: LOG.debug('release-type not given, ' 'guessing {!r}'.format(release_type)) # If this is a puppet module, ensure # that the tag and metadata file # match. if release_type == 'puppet': LOG.debug('applying puppet version rules') puppet_ver = puppetutils.get_version( context.workdir, project.repo.name) if puppet_ver != release.version: context.error( '%s metadata contains "%s" ' 'but is being tagged "%s"' % ( project.repo.name, puppet_ver, release.version, ) ) # If this is a npm module, ensure # that the tag and metadata file # match. if release_type == 'nodejs': LOG.debug('applying nodejs version rules') npm_ver = npmutils.get_version( context.workdir, project.repo.name) if npm_ver != release.version: context.error( '%s package.json contains "%s" ' 'but is being tagged "%s"' % ( project.repo.name, npm_ver, release.version, ) ) # If this is an xstatic package, ensure the package version in code # matches the tag being requested. if release_type == 'xstatic': LOG.debug('performing xstatic version checks') xs_versions = xstaticutils.get_versions( context.workdir, project.repo.name) if not xs_versions: context.error( '%s should contain a PACKAGE_VERSION but none found') for xs_version in xs_versions: if xs_version != release.version: context.error( '%s xstatic.pkg has PACKAGE_VERSION "%s" but is ' 'being tagged as "%s"' % ( project.repo.name, xs_version, release.version, ) ) # If we know the previous version and the # project is a python deliverable make sure # the requirements haven't changed in a way # not reflecting the version. if (prev_version.get(project.repo.name) and release_type in _PYTHON_RELEASE_TYPES): # For the master branch, enforce the # rules. For other branches just warn if # the rules are broken because there are # cases where we do need to support point # releases with requirements updates. if deliv.series == defaults.RELEASE: report = context.error else: report = context.warning requirements.find_bad_lower_bound_increases( context.workdir, project.repo.name, prev_version.get(project.repo.name), release.version, project.hash, report, ) had_error = False for e in versionutils.validate_version( release.version, release_type=release_type, pre_ok=(deliv.model in _USES_PREVER)): msg = ('could not validate version %r: %s' % (release.version, e)) context.error(msg) had_error = True if not had_error: print('{} for {} OK'.format( release.version, project.repo.name)) # Update the previous version information without discarding # any data about repositories that were not tagged in this # release. for project in release.projects: prev_version[project.repo.name] = release.version @skip_existing_tags @applies_to_released def validate_new_releases_at_end(deliv, context): "New releases must be added to the end of the list." # Remember which entries are new so we can verify that they # appear at the end of the file. new_releases = {} for release in deliv.releases: for project in release.projects: if not gitutils.checkout_ref(context.workdir, project.repo.name, project.hash, context): continue version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) if version_exists: print('tag exists, skipping further validation') continue LOG.debug('Found new version {} for {}'.format( release.version, project.repo)) new_releases[release.version] = release # Make sure that new entries have been appended to the file. for _, nr in new_releases.items(): LOG.debug('comparing {!r} to {!r}'.format(nr, deliv.releases[-1])) if nr != deliv.releases[-1]: msg = ('new release %s must be listed last, ' 'with one new release per patch' % nr.version) context.error(msg) else: print('OK') @skip_existing_tags @applies_to_released def validate_new_releases_in_open_series(deliv, context): "New releases may only be added to open series." if deliv.allows_releases: print('{} has status {!r} for {} and allows releases'.format( deliv.name, deliv.series, deliv.stable_status)) return LOG.debug('%s has status %r for %s and will not allow new releases', deliv.name, deliv.stable_status, deliv.series) # Remember which entries are new so we can verify that they # appear at the end of the file. new_releases = {} for release in deliv.releases: for project in release.projects: if not gitutils.checkout_ref(context.workdir, project.repo.name, project.hash, context): continue version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) if version_exists: print('tag exists, skipping further validation') continue if release.is_eol: LOG.debug('Found new EOL tag {} for {}'.format( release.version, project.repo)) elif release.is_em: LOG.debug('Found new EM tag {} for {}'.format( release.version, project.repo)) else: LOG.debug('Found new version {} for {}'.format( release.version, project.repo)) new_releases[release.version] = release if new_releases: # The series is closed but there is a new release. msg = ('deliverable {} has status {!r} for {} ' 'and cannot have new releases tagged').format( deliv.name, deliv.stable_status, deliv.series) context.error(msg) else: print('OK') @applies_to_released def validate_release_branch_membership(deliv, context): "Commits being tagged need to be on the right branch." if deliv.is_independent: context.warning('skipping descendant test for ' 'independent project, verify ' 'branch manually') return # Track the previous version tag attached to each repository, by # name. prev_version = {} for release in deliv.releases: LOG.debug('checking {}'.format(release.version)) for project in release.projects: if not gitutils.checkout_ref(context.workdir, project.repo.name, project.hash, context): continue version_exists = gitutils.commit_exists( context.workdir, project.repo.name, release.version, ) if version_exists: print('tag exists, skipping further validation') continue LOG.debug('Found new version {} for {}'.format( release.version, project.repo)) # If this is the first version in the series, # check that the commit is actually on the # targeted branch. if not gitutils.check_branch_sha(context.workdir, project.repo.name, deliv.series, project.hash): msg = '%s %s not present in %s branch' % ( project.repo.name, project.hash, deliv.series, ) context.error(msg) if not prev_version.get(project.repo.name): print('no ancestry check for first version in a series') continue # Check to see if we are re-tagging the same # commit with a new version. old_sha = gitutils.sha_for_tag( context.workdir, project.repo.name, prev_version[project.repo.name], ) if old_sha == project.hash: # FIXME(dhellmann): This needs a test. LOG.debug('The SHA is being retagged with a new version') else: # Check to see if the commit for the new # version is in the ancestors of the # previous release, meaning it is actually # merged into the branch. is_ancestor = gitutils.check_ancestry( context.workdir, project.repo.name, prev_version[project.repo.name], project.hash, ) if not is_ancestor: context.error( '%s %s receiving %s ' 'is not a descendant of %s' % ( project.repo.name, project.hash, release.version, prev_version[project.repo.name], ) ) else: print('ancestry OK') # Update the previous version information without discarding # any data about repositories that were not tagged in this # release. for project in release.projects: prev_version[project.repo.name] = release.version @applies_to_current @applies_to_released def validate_new_releases(deliv, context): "Apply validation rules that only apply to the current series." final_release = deliv.releases[-1] expected_repos = set( r.name for r in context.gov_data.get_repositories( deliverable_name=deliv.name, ) ) link_mode = deliv.artifact_link_mode if link_mode != 'none' and not expected_repos: context.error('unable to find deliverable %s in the governance list' % deliv.name) actual_repos = set( p.repo.name for p in final_release.projects ) for extra in actual_repos.difference(expected_repos): context.warning( 'release %s includes repository %s ' 'that is not in the governance list' % (final_release.version, extra) ) for missing in expected_repos.difference(actual_repos): context.warning( 'release %s is missing %s, ' 'which appears in the governance list: %s' % (final_release.version, missing, expected_repos) ) for repo in actual_repos: if repo not in deliv.known_repo_names: context.error( 'release %s includes repository %s ' 'that is not in the repository-settings section' % (final_release.version, repo) ) for missing in deliv.known_repo_names: if missing not in actual_repos: context.error( 'release %s is missing %s, ' 'which appears in the repository-settings list' % (final_release.version, missing) ) def validate_branch_prefixes(deliv, context): "Ensure all branch names have good prefixes." for branch in deliv.branches: if branch.prefix not in _VALID_BRANCH_PREFIXES: context.error('branch name %s does not use a valid prefix: %s' % ( branch.name, _VALID_BRANCH_PREFIXES)) def validate_stable_branches(deliv, context): "Apply the rules for stable branches." if deliv.launchpad_id in _NO_STABLE_BRANCH_CHECK: print('rule does not apply to this repo, skipping') return if deliv.type == 'tempest-plugin' and deliv.branches: context.error('Tempest plugins do not support branching.') return branch_mode = deliv.stable_branch_type known_series = sorted(list( d for d in os.listdir('deliverables') if not d.startswith('_') )) for branch in deliv.branches: try: prefix, series = branch.name.split('/') except ValueError: context.error( ('stable branch name expected to be stable/name ' 'but got %s') % (branch.name,)) continue if prefix != 'stable': print('{} is not a stable branch, skipping'.format( branch.name)) continue location = branch.location if branch_mode == 'std': if not isinstance(location, six.string_types): context.error( ('branch location for %s is ' 'expected to be a string but got a %s' % ( branch.name, type(location))) ) if not deliv.known_repo_names: context.error( ('Unable to validate branch {} for ' '{} without repository information').format( branch.name, deliv.name, ) ) return branch_exists = all( gitutils.stable_branch_exists( context.workdir, repo, series, ) for repo in deliv.known_repo_names ) if branch_exists: print('{} branch already exists, skipping validation'.format( branch.name)) continue if deliv.is_independent: print('"latest release" rule does not apply ' 'to independent repositories, skipping') else: latest_release = deliv.releases[-1] if location != latest_release.version: context.error( ('stable branches must be created from the latest ' 'tagged release, and %s for %s does not match %s' % ( location, branch.name, latest_release.version)) ) elif branch_mode == 'tagless': if not isinstance(location, dict): context.error( ('branch location for %s is ' 'expected to be a mapping but got a %s' % ( branch.name, type(location))) ) # The other rules aren't going to be testable, so skip them. continue for repo, loc in sorted(location.items()): if not is_a_hash(loc): context.error( ('tagless stable branches should be created ' 'from commits by SHA but location %s for ' 'branch %s of %s does not look ' 'like a SHA' % ( (loc, repo, branch.name))) ) # We can't clone the location if it isn't a SHA. continue if not gitutils.checkout_ref(context.workdir, repo, loc, context): continue if not gitutils.commit_exists(context.workdir, repo, loc): context.error( ('stable branches should be created from merged ' 'commits but location %s for branch %s of %s ' 'does not exist' % ( (loc, repo, branch.name))) ) elif branch_mode == 'upstream': if not isinstance(location, six.string_types): context.error( ('branch location for %s is ' 'expected to be a string but got a %s' % ( branch.name, type(location))) ) else: context.error( ('unrecognized stable-branch-type %r' % (branch_mode,)) ) if branch_mode == 'upstream': context.warning( 'skipping branch name check for upstream mode' ) elif deliv.is_independent: if series not in known_series: context.error( ('stable branches must be named for known series ' 'but %s was not found in %s' % ( branch.name, known_series)) ) else: if series != deliv.series: context.error( ('cycle-based projects must match series names ' 'for stable branches. %s should be stable/%s' % ( branch.name, deliv.series)) ) def validate_feature_branches(deliv, context): "Apply the rules for feature branches." if deliv.type == 'tempest-plugin' and deliv.branches: context.error('Tempest plugins do not support branching.') return for branch in deliv.branches: try: prefix, _ = branch.name.split('/') except ValueError: context.error( ('feature branch name expected to be feature/name ' 'but got %s') % (branch.name,)) continue if prefix != 'feature': print('{} is not a feature branch, skipping'.format( branch.name)) continue location = branch.location if not isinstance(location, dict): context.error( ('branch location for %s is ' 'expected to be a mapping but got a %s' % ( branch.name, type(location))) ) # The other rules aren't going to be testable, so skip them. continue for repo, loc in sorted(location.items()): if not is_a_hash(loc): context.error( ('feature branches should be created from commits by SHA ' 'but location %s for branch %s of %s does not look ' 'like a SHA' % ( (loc, repo, branch.name))) ) if not gitutils.commit_exists(context.workdir, repo, loc): context.error( ('feature branches should be created from merged commits ' 'but location %s for branch %s of %s does not exist' % ( (loc, repo, branch.name))) ) def validate_branch_points(deliv, context): "Make sure the branch points given are on the expected branches." # Check for 'upstream' branches. These track upstream release names and # do not align with OpenStack series names. if deliv.stable_branch_type == 'upstream': print('this project follows upstream branching conventions, skipping') return for branch in deliv.branches: LOG.debug('checking branch {!r}'.format(branch.name)) try: prefix, series = branch.name.split('/') except ValueError: print('could not parse the branch name, skipping') continue if prefix != 'stable': print('these rules do not apply to feature branches, skipping') continue expected = set([ 'master', branch.name, ]) location = branch.get_repo_map() for repo, hash in sorted(location.items()): LOG.debug('checking repo {}'.format(repo)) existing_branches = sorted([ (b.partition('/origin/')[-1] if b.startswith('remotes/origin/') else b) for b in gitutils.get_branches(context.workdir, repo) ]) # Remove the remote name prefix if it is present in the # branch name. containing = set( c.partition('/')[-1] if c.startswith('origin/') else c for c in gitutils.branches_containing( context.workdir, repo, hash) ) LOG.debug('found {} on branches {} in {}'.format( hash, containing, repo)) for missing in expected.difference(containing): if missing not in existing_branches: print('branch {} does not exist in {}, ' 'skipping'.format(branch.name, repo)) continue if branch.name in existing_branches: # The branch already exists but there is something # wrong with the specification. This probably # means someone tried to update the branch setting # after creating the branch, so phrase the error # message to reflect that. context.error( '{} branch exists in {} and does not seem ' 'to have been created from {}'.format( branch.name, repo, hash), ) else: # The branch does not exist and the proposed point # to create it is not on the expected source # branch, so phrase the error message to reflect # that. context.error( 'commit {} is not on the {} branch ' 'but it is listed as the branch point for ' '{} to be created'.format( hash, missing, branch.name)) # if the branch already exists, the name is by definition valid # if the branch exists, the data in the map must match reality # # FIXME(dhellmann): these two rules become more challenging to # implement when we think about EOLed branches. I'm going to punt on # that for now, and if it turns into an issue we can think about how # to handle validation while still allowing branches to be deleted. class ValidationContext(object): _zuul_projects = None _gov_data = None def __init__(self, debug=False, cleanup=True): self.warnings = [] self.errors = [] self.debug = debug self.cleanup = cleanup self.filename = None self._setup_workdir() self.function_name = 'unknown' def _setup_workdir(self): workdir = tempfile.mkdtemp(prefix='releases-') LOG.debug('creating temporary files in {}'.format(workdir)) def cleanup_workdir(): if self.cleanup: shutil.rmtree(workdir, True) else: print('not cleaning up %s' % workdir) atexit.register(cleanup_workdir) self.workdir = workdir def set_filename(self, filename): self.filename = filename def set_function(self, function): self.function_name = function.__name__ def warning(self, msg): LOG.warning(msg) self.warnings.append('{}: {}: {}'.format( self.filename, self.function_name, msg)) def error(self, msg): LOG.error(msg) self.errors.append('{}: {}: {}'.format( self.filename, self.function_name, msg)) if self.debug: raise RuntimeError(msg) def show_summary(self): header('Summary') print('\n\n%s warnings found' % len(self.warnings)) for w in self.warnings: print(w) print('\n\n%s errors found' % len(self.errors)) for e in self.errors: print(e) @property def zuul_projects(self): if not self._zuul_projects: self._zuul_projects = project_config.get_zuul_project_data() return self._zuul_projects @property def gov_data(self): if not self._gov_data: self._gov_data = governance.Governance.from_remote_repo() return self._gov_data def main(): parser = argparse.ArgumentParser() parser.add_argument( '--no-cleanup', dest='cleanup', default=True, action='store_false', help='do not remove temporary files', ) parser.add_argument( '--debug', default=False, action='store_true', help='throw exception on error', ) parser.add_argument( 'input', nargs='*', help=('YAML files to validate, defaults to ' 'files changed in the latest commit'), ) args = parser.parse_args() # Set up logging, including making some loggers quiet. logging.basicConfig( format='%(levelname)7s: %(message)s', stream=sys.stdout, level=logging.DEBUG, ) logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING) filenames = args.input or gitutils.find_modified_deliverable_files() if not filenames: LOG.warning('no modified deliverable files and no arguments, ' 'skipping validation') return 0 context = ValidationContext( debug=args.debug, cleanup=args.cleanup, ) for filename in filenames: header('Checking %s' % filename, '=') if not os.path.isfile(filename): print("File was deleted, skipping.") continue context.set_filename(filename) deliv = deliverable.Deliverable.read_file(filename) if deliv.series in _CLOSED_SERIES: print('File is part of a closed series, skipping') continue checks = [ clone_deliverable, validate_bugtracker, validate_team, validate_release_notes, validate_model, validate_release_type, validate_pypi_permissions, validate_pypi_readme, validate_build_sdist, validate_gitreview, validate_release_sha_exists, validate_existing_tags, validate_version_numbers, validate_new_releases_at_end, validate_new_releases_in_open_series, validate_release_branch_membership, validate_tarball_base, validate_new_releases, validate_series_open, validate_series_first, validate_series_final, validate_pre_release_progression, validate_series_eol, validate_series_em, validate_branch_prefixes, validate_stable_branches, validate_feature_branches, validate_branch_points, ] for check in checks: title = inspect.getdoc(check).splitlines()[0].strip() header(title) context.set_function(check) check(deliv, context) context.show_summary() return 1 if context.errors else 0