openstack/releases
Code Issues Proposed changes

update release job validation for zuulv3

Browse Source 
The release jobs are attached using a different configuration file
that has a different format. Update the validation logic to read the
new file and process its contents correctly.

Change-Id: Ib5ae569572b88524607bd320db6d746437d8636f
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2017-10-18 17:12:08 -04:00
parent 83cb36ec26
commit f066967e8d
3 changed files with 71 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
openstack_releases/cmds/validate.py
View File

@ -348,7 +348,7 @@ def validate_gitreview(deliverable_info, workdir, mk_warning, mk_error):
mk_error('%s has no .gitreview file' % (project['repo'],))
def validate_releases(deliverable_info, zuul_layout,
def validate_releases(deliverable_info, zuul_projects,
series_name,
workdir,
mk_warning, mk_error):
@ -381,7 +381,7 @@ def validate_releases(deliverable_info, zuul_layout,
# Check for release jobs (if we ship a tarball)
if link_mode != 'none':
project_config.require_release_jobs_for_repo(
deliverable_info, zuul_layout, project['repo'],
deliverable_info, zuul_projects, project['repo'],
release_type, mk_warning, mk_error,
)
@ -881,7 +881,7 @@ def main():
'skipping validation')
return 0
zuul_layout = project_config.get_zuul_layout_data()
zuul_projects = project_config.get_zuul_project_data()
team_data = governance.get_team_data()
@ -936,7 +936,7 @@ def main():
validate_gitreview(deliverable_info, workdir, mk_warning, mk_error)
validate_releases(
deliverable_info,
zuul_layout,
zuul_projects,
series_name,
workdir,
mk_warning,

70
openstack_releases/project_config.py
View File

@ -19,8 +19,8 @@ from openstack_releases import flags
from openstack_releases import yamlutils
ZUUL_LAYOUT_URL = 'http://git.openstack.org/cgit/openstack-infra/project-config/plain/zuul/layout.yaml' # noqa
ZUUL_LAYOUT_FILENAME = 'openstack-infra/project-config/zuul/layout.yaml'
ZUUL_PROJECTS_URL = 'http://git.openstack.org/cgit/openstack-infra/project-config/plain/zuul.d/projects.yaml' # noqa
ZUUL_PROJECTS_FILENAME = 'openstack-infra/project-config/zuul.d/projects.yaml'
# We use this key to modify the data structure read from the zuul
# layout file. We don't control what are valid keys there, so make it
@ -28,8 +28,8 @@ ZUUL_LAYOUT_FILENAME = 'openstack-infra/project-config/zuul/layout.yaml'
_VALIDATE_KEY = 'validate-projects-by-name'
def get_zuul_layout_data(url=ZUUL_LAYOUT_URL):
"""Return the parsed data structure for the zuul/layout.yaml file.
def get_zuul_project_data(url=ZUUL_PROJECTS_URL):
"""Return the data from the zuul.d/projects.yaml file.
:param url: Optional URL to the location of the file. Defaults to
the most current version in the public git repository.
@ -37,13 +37,46 @@ def get_zuul_layout_data(url=ZUUL_LAYOUT_URL):
"""
r = requests.get(url)
raw = yamlutils.loads(r.text)
# Add a mapping from repo name to repo settings, since that is how
# we access this most often.
raw[_VALIDATE_KEY] = {
p['name']: p
for p in raw['projects']
# Convert the raw list to a mapping from repo name to repo
# settings, since that is how we access this most often.
#
# The inputs are like:
#
# - project:
# name: openstack/oslo.config
# templates:
# - system-required
# - openstack-python-jobs
# - openstack-python35-jobs
# - publish-openstack-sphinx-docs
# - check-requirements
# - lib-forward-testing
# - release-notes-jobs
# - periodic-newton
# - periodic-ocata
# - periodic-pike
# - publish-to-pypi
#
# And the output is:
#
# openstack/oslo.config:
# templates:
# - system-required
# - openstack-python-jobs
# - openstack-python35-jobs
# - publish-openstack-sphinx-docs
# - check-requirements
# - lib-forward-testing
# - release-notes-jobs
# - periodic-newton
# - periodic-ocata
# - periodic-pike
# - publish-to-pypi
#
return {
p['project']['name']: p['project']
for p in raw
}
return raw
# Which jobs are needed for which release types.
@ -68,7 +101,7 @@ _RELEASE_JOBS_FOR_TYPE = {
}
def require_release_jobs_for_repo(deliverable_info, zuul_layout, repo,
def require_release_jobs_for_repo(deliverable_info, zuul_projects, repo,
release_type, mk_warning, mk_error):
"""Check the repository for release jobs.
@ -86,16 +119,13 @@ def require_release_jobs_for_repo(deliverable_info, zuul_layout, repo,
if flags.has_flag(deliverable_info, repo, flags.RETIRED):
return
if repo not in zuul_layout[_VALIDATE_KEY]:
if repo not in zuul_projects:
mk_error(
'did not find %s in %s' % (repo, ZUUL_LAYOUT_FILENAME),
'did not find %s in %s' % (repo, ZUUL_PROJECTS_FILENAME),
)
else:
p = zuul_layout[_VALIDATE_KEY][repo]
templates = [
t['name']
for t in p.get('template', [])
]
p = zuul_projects[repo]
templates = p.get('templates', [])
# NOTE(dhellmann): We don't mess around looking for individual
# jobs, because we want projects to use the templates.
expected_jobs = _RELEASE_JOBS_FOR_TYPE.get(
@ -111,12 +141,12 @@ def require_release_jobs_for_repo(deliverable_info, zuul_layout, repo,
mk_error(
'%s no release job specified for %s, '
'should be one of %r or no release will be '
'published' % (ZUUL_LAYOUT_FILENAME, repo, expected_jobs),
'published' % (ZUUL_PROJECTS_FILENAME, repo, expected_jobs),
)
elif num_release_jobs > 1:
mk_warning(
'%s multiple release jobs specified for %s, '
'should be *one* of %r'
% (ZUUL_LAYOUT_FILENAME, repo, expected_jobs),
% (ZUUL_PROJECTS_FILENAME, repo, expected_jobs),
)
return

34
openstack_releases/tests/test_project_config.py
View File

@ -29,13 +29,13 @@ class TestReleaseJobsStandard(base.BaseTestCase):
},
},
}
zuul_layout = {
zuul_projects = {
}
warnings = []
errors = []
project_config.require_release_jobs_for_repo(
deliverable_info,
{'validate-projects-by-name': zuul_layout},
{'validate-projects-by-name': zuul_projects},
'openstack/releases',
'std',
warnings.append,
@ -54,13 +54,13 @@ class TestReleaseJobsStandard(base.BaseTestCase):
}
}
}
zuul_layout = {
zuul_projects = {
}
warnings = []
errors = []
project_config.require_release_jobs_for_repo(
deliverable_info,
{'validate-projects-by-name': zuul_layout},
{'validate-projects-by-name': zuul_projects},
'openstack/releases',
'std',
warnings.append,
@ -69,16 +69,16 @@ class TestReleaseJobsStandard(base.BaseTestCase):
self.assertEqual(0, len(warnings))
self.assertEqual(0, len(errors))
def test_no_zuul_layout(self):
def test_no_zuul_projects(self):
deliverable_info = {
}
zuul_layout = {
zuul_projects = {
}
warnings = []
errors = []
project_config.require_release_jobs_for_repo(
deliverable_info,
{'validate-projects-by-name': zuul_layout},
{'validate-projects-by-name': zuul_projects},
'openstack/releases',
'std',
warnings.append,
@ -90,18 +90,18 @@ class TestReleaseJobsStandard(base.BaseTestCase):
def test_one_expected_job(self):
deliverable_info = {
}
zuul_layout = {
zuul_projects = {
'openstack/releases': {
'template': [
{'name': 'publish-to-pypi'},
'templates': [
'publish-to-pypi',
],
}
},
}
warnings = []
errors = []
project_config.require_release_jobs_for_repo(
deliverable_info,
{'validate-projects-by-name': zuul_layout},
zuul_projects,
'openstack/releases',
'std',
warnings.append,
@ -114,11 +114,11 @@ class TestReleaseJobsStandard(base.BaseTestCase):
def test_two_expected_jobs(self):
deliverable_info = {
}
zuul_layout = {
zuul_projects = {
'openstack/releases': {
'template': [
{'name': 'publish-to-pypi'},
{'name': 'puppet-tarball-jobs'},
'templates': [
'publish-to-pypi',
'puppet-tarball-jobs',
],
}
}
@ -126,7 +126,7 @@ class TestReleaseJobsStandard(base.BaseTestCase):
errors = []
project_config.require_release_jobs_for_repo(
deliverable_info,
{'validate-projects-by-name': zuul_layout},
zuul_projects,
'openstack/releases',
'std',
warnings.append,