Add projects.yaml support

The projects.yaml lists all of the "official" StarlingX projects
(the ones actually owned by StarlingX and not upstream forks).
It is in YAML format to be machine-readable and useful for a
number of tools.

The first such use is to generate project-specific pages in the
governance documentation listing the projects, tech leads and
project leads. The current content is basically taken from
the existing wiki pages and is lacking in certain areas.
Please review it carefully.

projects.py is based on the OpenStack governance teams.py.

Introduce hacking and switch to flake8 for Python linting,
and fix the pep8 errors in members.py.

Change-Id: I514de8d574df8e60bb488974f2796e2c41010363
Signed-off-by: Dean Troyer <dtroyer@gmail.com>

.gitignore

@@ -1,3 +1,7 @@
+# Only pay attention to the project index
+reference/tsc/projects/*.rst
+!reference/tsc/projects/index.rst
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

doc/requirements.txt

@@ -1,3 +1,4 @@
 openstack-doc-tools>=1.6.0
 openstackdocstheme>=1.19.0 # Apache-2.0
 sphinx>=1.6.2
+yamlordereddictloader

doc/source/_exts/members.py

@@ -15,8 +15,9 @@
 import re
 
 from docutils import nodes
-from docutils.parsers.rst.directives.tables import Table
 from docutils.parsers.rst import directives
+from docutils.parsers.rst.directives.tables import Table
+from docutils.utils import SystemMessagePropagation
 from sphinx.util import logging
 
 LOG = logging.getLogger(__name__)
@@ -64,7 +65,6 @@ class MembersTable(Table):
     def run(self):
         env = self.state.document.settings.env
         app = env.app
-        config = app.config
 
         # The required argument to the directive is the name of the
         # file to parse.
@@ -90,7 +90,7 @@ class MembersTable(Table):
                 'Error processing memberstable directive:\n%s' % err,
                 nodes.literal_block(self.block_text, self.block_text),
                 line=self.lineno,
-                )
+            )
             return [error]
 
         # Now find the real path to the file, relative to where we are.

doc/source/_exts/projects.py

@@ -0,0 +1,157 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Report on the current list of projects"""
+
+import copy
+from docutils import nodes
+from docutils.parsers import rst
+from docutils import statemachine
+import os
+from sphinx.util import logging
+from sphinx.util.nodes import nested_parse_with_titles
+import yaml
+import yamlordereddictloader
+
+
+LOG = logging.getLogger(__name__)
+
+IRC_LOG_URL_BASE = 'http://eavesdrop.openstack.org/irclogs/%23'
+
+_projects_yaml = {}
+
+
+def _get_project_data():
+    """Return a copy of the project data."""
+    return copy.deepcopy(_projects_yaml)
+
+
+def _load_project_file(filename='reference/tsc/projects.yaml'):
+    with open(filename, 'r', encoding='utf-8') as f:
+        return yaml.load(
+            f.read(),
+            Loader=yamlordereddictloader.Loader,
+        )
+
+
+def _project_to_rst(name, info):
+
+    if 'service' in info:
+        title = "{0} ({1})".format(name.title(), info['service'])
+    elif name == 'I18n':
+        title = name
+    else:
+        title = name.title()
+
+    yield '.. _project-%s:' % _slugify(name)
+    yield ''
+    yield '=' * len(title)
+    yield title
+    yield '=' * len(title)
+    yield ''
+    yield ':Home Page: ' + info.get('url', '')
+    tl = info.get('tl', {'name': '', 'irc': '', 'email': ''})
+    yield ':Technical Lead: %(name)s (``%(irc)s``) <%(email)s>' % tl
+    pl = info.get('pl', {'name': '', 'irc': '', 'email': ''})
+    yield ':Project Lead: %(name)s (``%(irc)s``) <%(email)s>' % pl
+    irc_channel = info.get('irc-channel')
+    if irc_channel:
+        yield ':IRC Channel: `#%s <%s%s>`__' % (
+            irc_channel, IRC_LOG_URL_BASE, irc_channel)
+    service = info.get('service')
+    if service:
+        yield ':Service: ' + service
+    yield ''
+    mission = info.get('mission', '').rstrip()
+    if mission:
+        yield "Mission"
+        yield '-------'
+        yield ''
+        yield mission
+        yield ''
+    yield 'Deliverables'
+    yield '------------'
+    yield ''
+    deliverables = info.get('deliverables', [])
+    if deliverables:
+        for repo_name, deliverable in deliverables.items():
+            yield repo_name
+            yield '~' * len(repo_name)
+            yield ''
+            yield ':Repositories: ' + ', '.join(
+                ':repo:`%s`' % repo
+                for repo in deliverable.get('repos', [])
+            )
+            yield ''
+            tags = deliverable.get('tags', [])
+            if tags:
+                yield ':Tags:'
+                yield ''
+                for tag in tags:
+                    yield '  - :ref:`tag-%s`' % tag
+                yield ''
+    else:
+        yield 'None'
+    yield ''
+
+
+def _slugify(name):
+    """Convert name to slug form for references."""
+    return name.lower().replace(' ', '-')
+
+
+def _write_project_pages(app):
+    all_projects = _get_project_data()
+    files = []
+    for project, info in all_projects.items():
+        LOG.info("project: %s" % project)
+        slug = _slugify(project)
+        filename = 'reference/tsc/projects/%s.rst' % slug
+        LOG.info('generating project page for %s' % project)
+        with open(filename, 'w', encoding='utf-8') as f:
+            f.write('\n'.join(_project_to_rst(project, info)))
+        files.append(filename)
+    return files
+
+
+class ProjectListDirective(rst.Directive):
+
+    has_content = False
+
+    def run(self):
+        all_projects = _get_project_data()
+
+        # Build the view of the data to be parsed for rendering.
+        result = statemachine.ViewList()
+        for project_name in sorted(all_projects.keys()):
+            project_info = all_projects[project_name]
+            for line in _project_to_rst(project_name, project_info):
+                result.append(line, '<' + __name__ + '>')
+
+        # Parse what we have into a new section.
+        node = nodes.section()
+        node.document = self.state.document
+        nested_parse_with_titles(self.state, result, node)
+
+        return node.children
+
+
+def setup(app):
+    global _projects_yaml
+
+    LOG.info('loading projects extension')
+    app.add_directive('projectlist', ProjectListDirective)
+
+    filename = os.path.abspath('reference/tsc/projects.yaml')
+    LOG.info('reading %s' % filename)
+    _projects_yaml = _load_project_file(filename)
+    _write_project_pages(app)

doc/source/conf.py

@@ -43,8 +43,15 @@ release = u'0.1'
 extensions = [
     'members',
     'openstackdocstheme',
+    'projects',
+    'sphinx.ext.extlinks',
 ]
 
+# Define shorthand roles for making links to common destinations.
+extlinks = {
+    'repo': ('https://git.startlingx.io/cgit/%s', ''),
+}
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 

reference/tsc/index.rst

@@ -17,6 +17,11 @@ documentation the links below are directly there.
 * `The Four Opens <https://governance.openstack.org/tc/reference/opens.html>`__
 * `StarlingX Charter <stx_charter.html>`__
 
+.. toctree::
+   :maxdepth: 1
+
+   projects/index
+
 Current Members
 ===============
 

reference/tsc/projects.yaml

@@ -0,0 +1,153 @@
+---
+config:
+  pl:
+    name: Dariush Eslimi
+    irc:
+    email: Dariush.Eslimi@windriver.com
+  tl:
+    name: John Kung
+    irc:
+    email: John.Kung@windriver.com
+  irc-channel: starlingx
+  service: Configuration
+  mission: >
+    Configure all the things
+  url: https://docs.starlingx.io/stx-config
+  deliverables:
+    config:
+      repos:
+        - stx-config
+
+distcloud:
+  pl:
+    name: Dariush Eslimi
+    irc:
+    email: Dariush.Eslimi@windriver.com
+  tl:
+    name: Bart Wensley
+    irc:
+    email: Barton.Wensley@windriver.com
+  irc-channel: starlingx
+  service: Distributed Cloud
+  mission: >
+    Spreading out the goodness
+  url: https://docs.starlingx.io/stx-distcloud
+  deliverables:
+    fault:
+      repos:
+        - stx-distcloud
+        - stx-distcloud-client
+
+fault:
+  pl:
+    name: Dariush Eslimi
+    irc:
+    email: Dariush.Eslimi@windriver.com
+  tl:
+    name: Tao Liu
+    irc:
+    email: Tao.Liu@windriver.com
+  irc-channel: starlingx
+  service: Fault Management
+  mission: >
+    Manage the faults
+  url: https://docs.starlingx.io/stx-fault
+  deliverables:
+    fault:
+      repos:
+        - stx-fault
+
+gui:
+  pl:
+    name: Dariush Eslimi
+    irc:
+    email: Dariush.Eslimi@windriver.com
+  tl:
+    name: Tyler Smith
+    irc:
+    email: Tyler.Smith@windriver.com
+  irc-channel: starlingx
+  service: Dashboard Plugin
+  mission: >
+    Web GUI
+  url: https://docs.starlingx.io/stx-gui
+  deliverables:
+    fault:
+      repos:
+        - stx-gui
+
+ha:
+  pl:
+    name: Dariush Eslimi
+    irc:
+    email: Dariush.Eslimi@windriver.com
+  tl:
+    name: Bin Quan
+    irc:
+    email: Bin.Qian@windriver.com
+  irc-channel: starlingx
+  service: High Availability
+  mission: >
+    Stayin' Alive
+  url: https://docs.starlingx.io/stx-ha
+  deliverables:
+    fault:
+      repos:
+        - stx-ha
+
+metal:
+  pl:
+    name: Ken Young
+    irc:
+    email: Ken.Young@windriver.com
+  tl:
+    name: Eric MacDonald
+    irc:
+    email: Eric.MacDonald@windriver.com
+  irc-channel: starlingx
+  service: Bare Metal Management
+  mission: >
+    Tin Cans 'R Us'
+  url: https://docs.starlingx.io/stx-metal
+  deliverables:
+    fault:
+      repos:
+        - stx-metal
+
+nfv:
+  pl:
+    name: Frank Miller
+    irc:
+    email: Frank.Miller@windriver.com
+  tl:
+    name: Bart Wensley
+    irc:
+    email: Barton.Wensley@windriver.com
+  irc-channel: starlingx
+  service: Network Function Virtualization
+  mission: >
+    Don't just stand there, do something!
+  url: https://docs.starlingx.io/stx-nfv
+  deliverables:
+    fault:
+      repos:
+        - stx-nfv
+
+update:
+  pl:
+    name: Dariush Eslimi
+    irc:
+    email: Dariush.Eslimi@windriver.com
+  tl:
+    name: Don Penney
+    irc:
+    email: Don.Penney@windriver.com
+  irc-channel: starlingx
+  service: Software Update
+  mission: >
+    Keeping up with the neighbors
+  url: https://docs.starlingx.io/stx-update
+  deliverables:
+    fault:
+      repos:
+        - stx-update

reference/tsc/projects/index.rst

@@ -0,0 +1,16 @@
+==================
+StarlingX Projects
+==================
+
+The projects of StarlingX
+
+.. _projects:
+
+Projects
+--------
+
+.. toctree::
+   :glob:
+   :titlesonly:
+
+   *

test-requirements.txt

@@ -1,2 +1,3 @@
+hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
 PyYAML>=3.1.0
 yamllint>=0.5.2

tools/check_projects_yaml_alphabetized.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -xe
+
+# Checks that reference/projects.yaml alphabetized and prints list of
+# projects that should be sorted.
+
+export TMPDIR=$(mktemp -d tox.XXXXXX)
+trap "rm -rf $TMPDIR" EXIT
+
+pushd $TMPDIR
+PROJECTS_LIST=$OLDPWD/reference/tsc/projects.yaml
+
+grep '^[a-zA-Z0-9]' $PROJECTS_LIST > projects_list
+
+LC_ALL=C sort --ignore-case projects_list -o projects_list.sorted
+
+if ! diff projects_list projects_list.sorted > projects_list.diff; then
+    echo "The following projects should be alphabetized: "
+    cat projects_list.diff | grep -e '> '
+    exit 1
+else
+    echo "Projects alphabetized"
+fi
+
+popd

tox.ini

@@ -27,13 +27,14 @@ commands =
             \( -name .tox -prune \)           \
             -o -type f -name '*.yaml'         \
             -print0 | xargs -0 yamllint"
+  {toxinidir}/tools/check_projects_yaml_alphabetized.sh
+  flake8
 
 [testenv:pep8]
 basepython = python3
 usedevelop = False
 skip_install = True
-deps = pep8
-commands = pep8
+commands = flake8
 
 [testenv:venv]
 basepython = python3