Organize documentation by subject area

This splits all of the current job and role documentation into
files by subject area so that jobs and roles are easier for users to
find.

This will require that any future new jobs or roles add a line to the
appropriate area of the documentation, since that can no longer be
done automatically.  A linter check is added to ensure that every
job and role continue to be documented.

After this refactor, we can begin to enhance the documentation pages
so that they include narrative documentation and subsections.

Change-Id: Ia6f0e89b57e3cb0d7d1745206384c946506d7ea0

doc/source/afs-roles.rst

@@ -0,0 +1,7 @@
+AFS Roles
+=========
+
+.. zuul:autorole:: destroy-afs-token
+.. zuul:autorole:: upload-afs
+.. zuul:autorole:: create-afs-token
+.. zuul:autorole:: release-afs-volume

doc/source/container-roles.rst

@@ -0,0 +1,15 @@
+Container Roles
+===============
+
+.. zuul:autorole:: build-docker-image
+.. zuul:autorole:: deploy-openshift
+.. zuul:autorole:: install-docker
+.. zuul:autorole:: install-kubernetes
+.. zuul:autorole:: install-openshift
+.. zuul:autorole:: promote-docker-image
+.. zuul:autorole:: pull-from-intermediate-registry
+.. zuul:autorole:: push-to-intermediate-registry
+.. zuul:autorole:: run-buildset-registry
+.. zuul:autorole:: upload-docker-image
+.. zuul:autorole:: use-buildset-registry
+.. zuul:autorole:: use-docker-mirror

doc/source/deprecated-roles.rst

@@ -0,0 +1,5 @@
+Deprecrated and Test Roles
+==========================
+
+.. zuul:autorole:: fetch-zuul-cloner
+.. zuul:autorole:: test-mirror-workspace-git-repos

doc/source/docker-jobs.rst

@@ -0,0 +1,6 @@
+Docker Jobs
+===========
+
+.. zuul:autojob:: build-docker-image
+.. zuul:autojob:: upload-docker-image
+.. zuul:autojob:: promote-docker-image

doc/source/galaxy-roles.rst

@@ -0,0 +1,4 @@
+Ansible Galaxy Roles
+====================
+
+.. zuul:autorole:: ansible-galaxy-import

doc/source/general-jobs.rst

@@ -0,0 +1,7 @@
+General Purpose Jobs
+====================
+
+.. zuul:autojob:: dco-license
+.. zuul:autojob:: unittests
+.. zuul:autojob:: multinode
+.. zuul:autojob:: run-test-command

doc/source/general-roles.rst

@@ -0,0 +1,36 @@
+General Purpose Roles
+=====================
+
+.. zuul:autorole:: add-authorized-keys
+.. zuul:autorole:: add-build-sshkey
+.. zuul:autorole:: add-gpgkey
+.. zuul:autorole:: add-sshkey
+.. zuul:autorole:: bindep
+.. zuul:autorole:: buildset-artifacts-location
+.. zuul:autorole:: configure-mirrors
+.. zuul:autorole:: copy-build-sshkey
+.. zuul:autorole:: download-artifact
+.. zuul:autorole:: emit-job-header
+.. zuul:autorole:: git-prepare-nodecache
+.. zuul:autorole:: log-inventory
+.. zuul:autorole:: mirror-workspace-git-repos
+.. zuul:autorole:: multi-node-bridge
+.. zuul:autorole:: multi-node-firewall
+.. zuul:autorole:: multi-node-hosts-file
+.. zuul:autorole:: multi-node-known-hosts
+.. zuul:autorole:: persistent-firewall
+.. zuul:autorole:: prepare-workspace
+.. zuul:autorole:: prepare-workspace-git
+.. zuul:autorole:: remove-build-sshkey
+.. zuul:autorole:: remove-gpgkey
+.. zuul:autorole:: remove-sshkey
+.. zuul:autorole:: revoke-sudo
+.. zuul:autorole:: sign-artifacts
+.. zuul:autorole:: stage-output
+.. zuul:autorole:: start-zuul-console
+.. zuul:autorole:: test-setup
+.. zuul:autorole:: trigger-readthedocs
+.. zuul:autorole:: validate-dco-license
+.. zuul:autorole:: validate-host
+.. zuul:autorole:: version-from-git
+.. zuul:autorole:: write-inventory

doc/source/index.rst

@@ -1,7 +1,7 @@
 .. include:: ../../README.rst
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    install
    policy

doc/source/jobs.rst

@@ -1,5 +1,10 @@
 Jobs
-=====
+====
 
-.. zuul:autojobs::
+.. toctree::
+   :maxdepth: 1
 
+   general-jobs
+   python-jobs
+   js-jobs
+   docker-jobs

doc/source/js-jobs.rst

@@ -0,0 +1,9 @@
+Javascript Jobs
+===============
+
+.. zuul:autojob:: build-javascript-tarball
+.. zuul:autojob:: build-javascript-content
+.. zuul:autojob:: nodejs-npm
+.. zuul:autojob:: nodejs-npm-run-test
+.. zuul:autojob:: nodejs-npm-run-lint
+.. zuul:autojob:: nodejs-npm-run-docs

doc/source/js-roles.rst

@@ -0,0 +1,13 @@
+Javascript Roles
+================
+
+.. zuul:autorole:: fetch-javascript-content-tarball
+.. zuul:autorole:: fetch-javascript-output
+.. zuul:autorole:: fetch-javascript-tarball
+.. zuul:autorole:: install-javascript-packages
+.. zuul:autorole:: install-nodejs
+.. zuul:autorole:: install-yarn
+.. zuul:autorole:: nodejs-test-dependencies
+.. zuul:autorole:: npm
+.. zuul:autorole:: upload-npm
+.. zuul:autorole:: yarn

doc/source/launchpad-roles.rst

@@ -0,0 +1,6 @@
+Launchpad Roles
+===============
+
+.. zuul:autorole:: add-launchpad-credentials
+.. zuul:autorole:: remove-launchpad-credentials
+

doc/source/log-roles.rst

@@ -0,0 +1,13 @@
+Log Roles
+=========
+
+.. zuul:autorole:: add-fileserver
+.. zuul:autorole:: ara-report
+.. zuul:autorole:: ensure-output-dirs
+.. zuul:autorole:: fetch-output
+.. zuul:autorole:: htmlify-logs
+.. zuul:autorole:: merge-output-to-logs
+.. zuul:autorole:: publish-artifacts-to-fileserver
+.. zuul:autorole:: set-zuul-log-path-fact
+.. zuul:autorole:: upload-logs
+.. zuul:autorole:: upload-logs-swift

doc/source/puppet-roles.rst

@@ -0,0 +1,6 @@
+Puppet Roles
+============
+
+.. zuul:autorole:: build-puppet-module
+.. zuul:autorole:: fetch-puppet-module-output
+.. zuul:autorole:: upload-forge

doc/source/python-jobs.rst

@@ -0,0 +1,18 @@
+Python Jobs
+===========
+
+.. zuul:autojob:: tox
+.. zuul:autojob:: tox-py27
+.. zuul:autojob:: tox-py34
+.. zuul:autojob:: tox-py35
+.. zuul:autojob:: tox-py36
+.. zuul:autojob:: tox-docs
+.. zuul:autojob:: tox-linters
+.. zuul:autojob:: tox-pep8
+.. zuul:autojob:: tox-cover
+.. zuul:autojob:: tox-bashate
+.. zuul:autojob:: tox-nodejs-npm
+.. zuul:autojob:: build-python-release
+.. zuul:autojob:: python-upload-pypi
+.. zuul:autojob:: build-sphinx-docs
+.. zuul:autojob:: build-reno-releasenotes

doc/source/python-roles.rst

@@ -0,0 +1,21 @@
+Python Roles
+============
+
+.. zuul:autorole:: build-python-release
+.. zuul:autorole:: build-releasenotes
+.. zuul:autorole:: ensure-babel
+.. zuul:autorole:: ensure-python
+.. zuul:autorole:: ensure-sphinx
+.. zuul:autorole:: ensure-tox
+.. zuul:autorole:: ensure-twine
+.. zuul:autorole:: fetch-coverage-output
+.. zuul:autorole:: fetch-python-sdist-output
+.. zuul:autorole:: fetch-sphinx-output
+.. zuul:autorole:: fetch-sphinx-tarball
+.. zuul:autorole:: fetch-subunit-output
+.. zuul:autorole:: fetch-tox-output
+.. zuul:autorole:: find-constraints
+.. zuul:autorole:: install-if-python
+.. zuul:autorole:: sphinx
+.. zuul:autorole:: tox
+.. zuul:autorole:: upload-pypi

doc/source/roles.rst

@@ -1,4 +1,17 @@
 Roles
 =====
 
-.. zuul:autoroles::
+.. toctree::
+   :maxdepth: 1
+
+   general-roles
+   log-roles
+   afs-roles
+   container-roles
+   deprecated-roles
+   galaxy-roles
+   js-roles
+   launchpad-roles
+   puppet-roles
+   python-roles
+   translation-roles

doc/source/translation-roles.rst

@@ -0,0 +1,4 @@
+Translation Roles
+=================
+
+.. zuul:autorole:: fetch-translation-output

tools/check_jobs_documented.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+#
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Ensure that all jobs and roles appear in the documentation.
+
+import os
+import re
+import sys
+import yaml
+
+
+class ZuulSafeLoader(yaml.SafeLoader):
+
+    def __init__(self, *args, **kwargs):
+        super(ZuulSafeLoader, self).__init__(*args, **kwargs)
+        self.add_multi_constructor('!encrypted/', self.construct_encrypted)
+
+    @classmethod
+    def construct_encrypted(cls, loader, tag_suffix, node):
+        return loader.construct_sequence(node)
+
+
+class Layout(object):
+    def __init__(self):
+        self.jobs = []
+
+
+class ZuulConfig(object):
+    def find_zuul_yaml(self):
+        root = os.getcwd()
+        while root:
+            for fn in ['zuul.yaml', '.zuul.yaml', 'zuul.d', '.zuul.d']:
+                path = os.path.join(root, fn)
+                if os.path.exists(path):
+                    return path
+            root = os.path.split(root)[0]
+        raise Exception(
+            "Unable to find zuul config in zuul.yaml, .zuul.yaml,"
+            " zuul.d or .zuul.d")
+
+    def parse_zuul_yaml(self, path):
+        with open(path) as f:
+            data = yaml.load(f, Loader=ZuulSafeLoader)
+        layout = Layout()
+        for obj in data:
+            if 'job' in obj:
+                layout.jobs.append(obj['job'])
+        return layout
+
+    def parse_zuul_d(self, path):
+        layout = Layout()
+        for conf in os.listdir(path):
+            with open(os.path.join(path, conf)) as f:
+                data = yaml.load(f, Loader=ZuulSafeLoader)
+            for obj in data:
+                if 'job' in obj:
+                    layout.jobs.append(obj['job'])
+        return layout
+
+    def parse_zuul_layout(self):
+        path = self.find_zuul_yaml()
+        if path.endswith('zuul.d'):
+            layout = self.parse_zuul_d(path)
+        else:
+            layout = self.parse_zuul_yaml(path)
+        return layout
+
+    def __init__(self):
+        self.layout = self.parse_zuul_layout()
+
+
+class Docs(object):
+    def __init__(self):
+        self.jobs = set()
+        self.roles = set()
+        self.autojobs = False
+        self.autoroles = False
+        self.walk(os.path.join(os.getcwd(), 'doc', 'source'))
+
+    def walk(self, path):
+        for root, dirs, files in os.walk(path):
+            for fn in files:
+                if fn.endswith('.rst'):
+                    with open(os.path.join(root, fn)) as f:
+                        for line in f:
+                            m = re.match(r'.*\.\. zuul:job:: (.*)$', line)
+                            if m:
+                                self.jobs.add(m.group(1))
+                            m = re.match(r'.*\.\. zuul:autojob:: (.*)$', line)
+                            if m:
+                                self.jobs.add(m.group(1))
+                            m = re.match(r'.*\.\. zuul:autojobs::.*$', line)
+                            if m:
+                                self.autojobs = True
+                            m = re.match(r'.*\.\. zuul:role:: (.*)$', line)
+                            if m:
+                                self.roles.add(m.group(1))
+                            m = re.match(r'.*\.\. zuul:autorole:: (.*)$', line)
+                            if m:
+                                self.roles.add(m.group(1))
+                            m = re.match(r'.*\.\. zuul:autoroles::.*$', line)
+                            if m:
+                                self.autoroles = True
+
+
+class Roles(object):
+    def __init__(self):
+        self.roles = set()
+        self.walk(os.path.join(os.getcwd(), 'roles'))
+
+    def walk(self, path):
+        for role in os.listdir(path):
+            if os.path.isdir(os.path.join(path, role, 'tasks')):
+                self.roles.add(role)
+
+
+z = ZuulConfig()
+r = Roles()
+d = Docs()
+
+ret = 0
+for role in r.roles:
+    if role not in d.roles:
+        print("Role %s not included in document tree" % (role,))
+        ret = 1
+for job in [x['name'] for x in z.layout.jobs]:
+    if job not in d.jobs:
+        print("Job %s not included in document tree" % (job,))
+        ret = 1
+
+sys.exit(ret)

tox.ini

@@ -46,6 +46,7 @@ commands =
   # Ansible Syntax Check
   bash -c "find playbooks -type f -regex '.*.ya?ml' -exec \
     ansible-playbook --syntax-check -i {toxinidir}/tests/inventory \{\} + > /dev/null"
+  {toxinidir}/tools/check_jobs_documented.py
 
 [testenv:venv]
 commands = {posargs}