diff --git a/doc/source/afs-roles.rst b/doc/source/afs-roles.rst new file mode 100644 index 000000000..c2e15e64d --- /dev/null +++ b/doc/source/afs-roles.rst @@ -0,0 +1,7 @@ +AFS Roles +========= + +.. zuul:autorole:: destroy-afs-token +.. zuul:autorole:: upload-afs +.. zuul:autorole:: create-afs-token +.. zuul:autorole:: release-afs-volume diff --git a/doc/source/container-roles.rst b/doc/source/container-roles.rst new file mode 100644 index 000000000..2e2feb360 --- /dev/null +++ b/doc/source/container-roles.rst @@ -0,0 +1,15 @@ +Container Roles +=============== + +.. zuul:autorole:: build-docker-image +.. zuul:autorole:: deploy-openshift +.. zuul:autorole:: install-docker +.. zuul:autorole:: install-kubernetes +.. zuul:autorole:: install-openshift +.. zuul:autorole:: promote-docker-image +.. zuul:autorole:: pull-from-intermediate-registry +.. zuul:autorole:: push-to-intermediate-registry +.. zuul:autorole:: run-buildset-registry +.. zuul:autorole:: upload-docker-image +.. zuul:autorole:: use-buildset-registry +.. zuul:autorole:: use-docker-mirror diff --git a/doc/source/deprecated-roles.rst b/doc/source/deprecated-roles.rst new file mode 100644 index 000000000..89fc76a39 --- /dev/null +++ b/doc/source/deprecated-roles.rst @@ -0,0 +1,5 @@ +Deprecrated and Test Roles +========================== + +.. zuul:autorole:: fetch-zuul-cloner +.. zuul:autorole:: test-mirror-workspace-git-repos diff --git a/doc/source/docker-jobs.rst b/doc/source/docker-jobs.rst new file mode 100644 index 000000000..d77d97b52 --- /dev/null +++ b/doc/source/docker-jobs.rst @@ -0,0 +1,6 @@ +Docker Jobs +=========== + +.. zuul:autojob:: build-docker-image +.. zuul:autojob:: upload-docker-image +.. zuul:autojob:: promote-docker-image diff --git a/doc/source/galaxy-roles.rst b/doc/source/galaxy-roles.rst new file mode 100644 index 000000000..e7f185157 --- /dev/null +++ b/doc/source/galaxy-roles.rst @@ -0,0 +1,4 @@ +Ansible Galaxy Roles +==================== + +.. zuul:autorole:: ansible-galaxy-import diff --git a/doc/source/general-jobs.rst b/doc/source/general-jobs.rst new file mode 100644 index 000000000..314e093bd --- /dev/null +++ b/doc/source/general-jobs.rst @@ -0,0 +1,7 @@ +General Purpose Jobs +==================== + +.. zuul:autojob:: dco-license +.. zuul:autojob:: unittests +.. zuul:autojob:: multinode +.. zuul:autojob:: run-test-command diff --git a/doc/source/general-roles.rst b/doc/source/general-roles.rst new file mode 100644 index 000000000..7a18505a8 --- /dev/null +++ b/doc/source/general-roles.rst @@ -0,0 +1,36 @@ +General Purpose Roles +===================== + +.. zuul:autorole:: add-authorized-keys +.. zuul:autorole:: add-build-sshkey +.. zuul:autorole:: add-gpgkey +.. zuul:autorole:: add-sshkey +.. zuul:autorole:: bindep +.. zuul:autorole:: buildset-artifacts-location +.. zuul:autorole:: configure-mirrors +.. zuul:autorole:: copy-build-sshkey +.. zuul:autorole:: download-artifact +.. zuul:autorole:: emit-job-header +.. zuul:autorole:: git-prepare-nodecache +.. zuul:autorole:: log-inventory +.. zuul:autorole:: mirror-workspace-git-repos +.. zuul:autorole:: multi-node-bridge +.. zuul:autorole:: multi-node-firewall +.. zuul:autorole:: multi-node-hosts-file +.. zuul:autorole:: multi-node-known-hosts +.. zuul:autorole:: persistent-firewall +.. zuul:autorole:: prepare-workspace +.. zuul:autorole:: prepare-workspace-git +.. zuul:autorole:: remove-build-sshkey +.. zuul:autorole:: remove-gpgkey +.. zuul:autorole:: remove-sshkey +.. zuul:autorole:: revoke-sudo +.. zuul:autorole:: sign-artifacts +.. zuul:autorole:: stage-output +.. zuul:autorole:: start-zuul-console +.. zuul:autorole:: test-setup +.. zuul:autorole:: trigger-readthedocs +.. zuul:autorole:: validate-dco-license +.. zuul:autorole:: validate-host +.. zuul:autorole:: version-from-git +.. zuul:autorole:: write-inventory diff --git a/doc/source/index.rst b/doc/source/index.rst index 31017c326..a802f395f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,7 +1,7 @@ .. include:: ../../README.rst .. toctree:: - :maxdepth: 1 + :maxdepth: 2 install policy diff --git a/doc/source/jobs.rst b/doc/source/jobs.rst index 1d722cb91..b1141eb86 100644 --- a/doc/source/jobs.rst +++ b/doc/source/jobs.rst @@ -1,5 +1,10 @@ Jobs -===== +==== -.. zuul:autojobs:: +.. toctree:: + :maxdepth: 1 + general-jobs + python-jobs + js-jobs + docker-jobs diff --git a/doc/source/js-jobs.rst b/doc/source/js-jobs.rst new file mode 100644 index 000000000..200648466 --- /dev/null +++ b/doc/source/js-jobs.rst @@ -0,0 +1,9 @@ +Javascript Jobs +=============== + +.. zuul:autojob:: build-javascript-tarball +.. zuul:autojob:: build-javascript-content +.. zuul:autojob:: nodejs-npm +.. zuul:autojob:: nodejs-npm-run-test +.. zuul:autojob:: nodejs-npm-run-lint +.. zuul:autojob:: nodejs-npm-run-docs diff --git a/doc/source/js-roles.rst b/doc/source/js-roles.rst new file mode 100644 index 000000000..0c628d84d --- /dev/null +++ b/doc/source/js-roles.rst @@ -0,0 +1,13 @@ +Javascript Roles +================ + +.. zuul:autorole:: fetch-javascript-content-tarball +.. zuul:autorole:: fetch-javascript-output +.. zuul:autorole:: fetch-javascript-tarball +.. zuul:autorole:: install-javascript-packages +.. zuul:autorole:: install-nodejs +.. zuul:autorole:: install-yarn +.. zuul:autorole:: nodejs-test-dependencies +.. zuul:autorole:: npm +.. zuul:autorole:: upload-npm +.. zuul:autorole:: yarn diff --git a/doc/source/launchpad-roles.rst b/doc/source/launchpad-roles.rst new file mode 100644 index 000000000..051d3cd1f --- /dev/null +++ b/doc/source/launchpad-roles.rst @@ -0,0 +1,6 @@ +Launchpad Roles +=============== + +.. zuul:autorole:: add-launchpad-credentials +.. zuul:autorole:: remove-launchpad-credentials + diff --git a/doc/source/log-roles.rst b/doc/source/log-roles.rst new file mode 100644 index 000000000..395f2426b --- /dev/null +++ b/doc/source/log-roles.rst @@ -0,0 +1,13 @@ +Log Roles +========= + +.. zuul:autorole:: add-fileserver +.. zuul:autorole:: ara-report +.. zuul:autorole:: ensure-output-dirs +.. zuul:autorole:: fetch-output +.. zuul:autorole:: htmlify-logs +.. zuul:autorole:: merge-output-to-logs +.. zuul:autorole:: publish-artifacts-to-fileserver +.. zuul:autorole:: set-zuul-log-path-fact +.. zuul:autorole:: upload-logs +.. zuul:autorole:: upload-logs-swift diff --git a/doc/source/puppet-roles.rst b/doc/source/puppet-roles.rst new file mode 100644 index 000000000..7ac9e64e0 --- /dev/null +++ b/doc/source/puppet-roles.rst @@ -0,0 +1,6 @@ +Puppet Roles +============ + +.. zuul:autorole:: build-puppet-module +.. zuul:autorole:: fetch-puppet-module-output +.. zuul:autorole:: upload-forge diff --git a/doc/source/python-jobs.rst b/doc/source/python-jobs.rst new file mode 100644 index 000000000..2f39cbd20 --- /dev/null +++ b/doc/source/python-jobs.rst @@ -0,0 +1,18 @@ +Python Jobs +=========== + +.. zuul:autojob:: tox +.. zuul:autojob:: tox-py27 +.. zuul:autojob:: tox-py34 +.. zuul:autojob:: tox-py35 +.. zuul:autojob:: tox-py36 +.. zuul:autojob:: tox-docs +.. zuul:autojob:: tox-linters +.. zuul:autojob:: tox-pep8 +.. zuul:autojob:: tox-cover +.. zuul:autojob:: tox-bashate +.. zuul:autojob:: tox-nodejs-npm +.. zuul:autojob:: build-python-release +.. zuul:autojob:: python-upload-pypi +.. zuul:autojob:: build-sphinx-docs +.. zuul:autojob:: build-reno-releasenotes diff --git a/doc/source/python-roles.rst b/doc/source/python-roles.rst new file mode 100644 index 000000000..f7c8aacfc --- /dev/null +++ b/doc/source/python-roles.rst @@ -0,0 +1,21 @@ +Python Roles +============ + +.. zuul:autorole:: build-python-release +.. zuul:autorole:: build-releasenotes +.. zuul:autorole:: ensure-babel +.. zuul:autorole:: ensure-python +.. zuul:autorole:: ensure-sphinx +.. zuul:autorole:: ensure-tox +.. zuul:autorole:: ensure-twine +.. zuul:autorole:: fetch-coverage-output +.. zuul:autorole:: fetch-python-sdist-output +.. zuul:autorole:: fetch-sphinx-output +.. zuul:autorole:: fetch-sphinx-tarball +.. zuul:autorole:: fetch-subunit-output +.. zuul:autorole:: fetch-tox-output +.. zuul:autorole:: find-constraints +.. zuul:autorole:: install-if-python +.. zuul:autorole:: sphinx +.. zuul:autorole:: tox +.. zuul:autorole:: upload-pypi diff --git a/doc/source/roles.rst b/doc/source/roles.rst index 5baa1e4e8..debc6534d 100644 --- a/doc/source/roles.rst +++ b/doc/source/roles.rst @@ -1,4 +1,17 @@ Roles ===== -.. zuul:autoroles:: +.. toctree:: + :maxdepth: 1 + + general-roles + log-roles + afs-roles + container-roles + deprecated-roles + galaxy-roles + js-roles + launchpad-roles + puppet-roles + python-roles + translation-roles diff --git a/doc/source/translation-roles.rst b/doc/source/translation-roles.rst new file mode 100644 index 000000000..e81a3f439 --- /dev/null +++ b/doc/source/translation-roles.rst @@ -0,0 +1,4 @@ +Translation Roles +================= + +.. zuul:autorole:: fetch-translation-output diff --git a/tools/check_jobs_documented.py b/tools/check_jobs_documented.py new file mode 100755 index 000000000..d39c0c9e8 --- /dev/null +++ b/tools/check_jobs_documented.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Ensure that all jobs and roles appear in the documentation. + +import os +import re +import sys +import yaml + + +class ZuulSafeLoader(yaml.SafeLoader): + + def __init__(self, *args, **kwargs): + super(ZuulSafeLoader, self).__init__(*args, **kwargs) + self.add_multi_constructor('!encrypted/', self.construct_encrypted) + + @classmethod + def construct_encrypted(cls, loader, tag_suffix, node): + return loader.construct_sequence(node) + + +class Layout(object): + def __init__(self): + self.jobs = [] + + +class ZuulConfig(object): + def find_zuul_yaml(self): + root = os.getcwd() + while root: + for fn in ['zuul.yaml', '.zuul.yaml', 'zuul.d', '.zuul.d']: + path = os.path.join(root, fn) + if os.path.exists(path): + return path + root = os.path.split(root)[0] + raise Exception( + "Unable to find zuul config in zuul.yaml, .zuul.yaml," + " zuul.d or .zuul.d") + + def parse_zuul_yaml(self, path): + with open(path) as f: + data = yaml.load(f, Loader=ZuulSafeLoader) + layout = Layout() + for obj in data: + if 'job' in obj: + layout.jobs.append(obj['job']) + return layout + + def parse_zuul_d(self, path): + layout = Layout() + for conf in os.listdir(path): + with open(os.path.join(path, conf)) as f: + data = yaml.load(f, Loader=ZuulSafeLoader) + for obj in data: + if 'job' in obj: + layout.jobs.append(obj['job']) + return layout + + def parse_zuul_layout(self): + path = self.find_zuul_yaml() + if path.endswith('zuul.d'): + layout = self.parse_zuul_d(path) + else: + layout = self.parse_zuul_yaml(path) + return layout + + def __init__(self): + self.layout = self.parse_zuul_layout() + + +class Docs(object): + def __init__(self): + self.jobs = set() + self.roles = set() + self.autojobs = False + self.autoroles = False + self.walk(os.path.join(os.getcwd(), 'doc', 'source')) + + def walk(self, path): + for root, dirs, files in os.walk(path): + for fn in files: + if fn.endswith('.rst'): + with open(os.path.join(root, fn)) as f: + for line in f: + m = re.match(r'.*\.\. zuul:job:: (.*)$', line) + if m: + self.jobs.add(m.group(1)) + m = re.match(r'.*\.\. zuul:autojob:: (.*)$', line) + if m: + self.jobs.add(m.group(1)) + m = re.match(r'.*\.\. zuul:autojobs::.*$', line) + if m: + self.autojobs = True + m = re.match(r'.*\.\. zuul:role:: (.*)$', line) + if m: + self.roles.add(m.group(1)) + m = re.match(r'.*\.\. zuul:autorole:: (.*)$', line) + if m: + self.roles.add(m.group(1)) + m = re.match(r'.*\.\. zuul:autoroles::.*$', line) + if m: + self.autoroles = True + + +class Roles(object): + def __init__(self): + self.roles = set() + self.walk(os.path.join(os.getcwd(), 'roles')) + + def walk(self, path): + for role in os.listdir(path): + if os.path.isdir(os.path.join(path, role, 'tasks')): + self.roles.add(role) + + +z = ZuulConfig() +r = Roles() +d = Docs() + +ret = 0 +for role in r.roles: + if role not in d.roles: + print("Role %s not included in document tree" % (role,)) + ret = 1 +for job in [x['name'] for x in z.layout.jobs]: + if job not in d.jobs: + print("Job %s not included in document tree" % (job,)) + ret = 1 + +sys.exit(ret) diff --git a/tox.ini b/tox.ini index b53e98bad..6f30ba105 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,7 @@ commands = # Ansible Syntax Check bash -c "find playbooks -type f -regex '.*.ya?ml' -exec \ ansible-playbook --syntax-check -i {toxinidir}/tests/inventory \{\} + > /dev/null" + {toxinidir}/tools/check_jobs_documented.py [testenv:venv] commands = {posargs}