add module for fetching and parsing governance data

Change-Id: I7279023e5d81554e19f315fe12ab6fdb988af934
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

.gitignore

@@ -6,3 +6,7 @@ reference/projects/*.rst
 *.pyc
 AUTHORS
 ChangeLog
+.stestr
+.coverage.*
+.coverage
+cover/

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=./openstack_governance/tests
+top_dir=./

.zuul.yaml

@@ -1,6 +1,7 @@
 - project:
     templates:
       - build-openstack-docs-pti
+      - openstack-python36-jobs
     check:
       jobs:
         - openstack-tox-linters:

openstack_governance/governance.py

@@ -0,0 +1,149 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Work with the governance repository.
+"""
+
+import weakref
+
+from openstack_governance import yamlutils
+
+import requests
+
+PROJECTS_LIST = "http://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml"  # noqa
+
+
+def get_team_data(url=PROJECTS_LIST):
+    """Return the parsed team data from the governance repository.
+
+    :param url: Optional URL to the location of the projects.yaml
+        file. Defaults to the most current version in the public git
+        repository.
+
+    """
+    r = requests.get(url)
+    return yamlutils.loads(r.text)
+
+
+def get_tags_for_deliverable(team_data, team, name):
+    "Return the tags for the deliverable owned by the team."
+    if team not in team_data:
+        return set()
+    team_info = team_data[team]
+    dinfo = team_info['deliverables'].get(name)
+    if not dinfo:
+        return set()
+    return set(dinfo.get('tags', [])).union(set(team_info.get('tags', [])))
+
+
+def get_repo_owner(team_data, repo_name):
+    """Return the name of the team that owns the repository.
+
+    :param team_data: The result of calling :func:`get_team_data`
+    :param repo_name: Long name of the repository, such as 'openstack/nova'.
+
+    """
+    for team, info in team_data.items():
+        for dname, dinfo in info.get('deliverables', {}).items():
+            if repo_name in dinfo.get('repos', []):
+                return team
+    raise ValueError('Repository %s not found in governance list' % repo_name)
+
+
+class Team(object):
+    _liaison_data = None
+
+    def __init__(self, name, data):
+        self.name = name
+        self.data = data
+        # Protectively initialize the ptl data structure in case part
+        # of it is missing from the project list, then replace any
+        # values we do have from that data.
+        self.ptl = {
+            'name': 'MISSING',
+            'irc': 'MISSING',
+        }
+        self.ptl.update(data.get('ptl', {}))
+        self.deliverables = {
+            dn: Deliverable(dn, di, self)
+            for dn, di in self.data.get('deliverables', {}).items()
+        }
+
+    @property
+    def tags(self):
+        return set(self.data.get('tags', []))
+
+
+class Deliverable(object):
+    def __init__(self, name, data, team):
+        self.name = name
+        self.data = data
+        self.team = weakref.proxy(team)
+        self.repositories = {
+            rn: Repository(rn, self)
+            for rn in self.data.get('repos', [])
+        }
+
+    @property
+    def tags(self):
+        return set(self.data.get('tags', [])).union(self.team.tags)
+
+
+class Repository(object):
+    def __init__(self, name, deliverable):
+        self.name = name
+        self.deliverable = weakref.proxy(deliverable)
+
+    @property
+    def tags(self):
+        return self.deliverable.tags
+
+
+def get_repositories(team_data, team_name=None, deliverable_name=None,
+                     tags=[], code_only=False):
+    """Return a sequence of repositories, possibly filtered.
+
+    :param team_data: The result of calling :func:`get_team_data`
+    :param team_name: The name of the team owning the repositories. Can be
+        None.
+    :para deliverable_name: The name of the deliverable to which all
+       repos should belong.
+    :param tags: The names of any tags the repositories should
+        have. Can be empty.
+    :param code_only: Boolean indicating whether to return only code
+      repositories (ignoring specs and cookiecutter templates).
+
+    """
+    if tags:
+        tags = set(tags)
+    if team_name:
+        try:
+            teams = [Team(team_name, team_data[team_name])]
+        except KeyError:
+            raise RuntimeError('No team %r found in %r' %
+                               (team_name, list(team_data.keys())))
+    else:
+        teams = [Team(n, i) for n, i in team_data.items()]
+    for team in teams:
+        if deliverable_name and deliverable_name not in team.deliverables:
+            continue
+        if deliverable_name:
+            deliverables = [team.deliverables[deliverable_name]]
+        else:
+            deliverables = team.deliverables.values()
+        for deliverable in deliverables:
+            for repository in deliverable.repositories.values():
+                if tags and not tags.issubset(repository.tags):
+                    continue
+                if code_only and not repository.code_related:
+                    continue
+                yield repository

openstack_governance/tests/test_governance.py

@@ -0,0 +1,127 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslotest import base
+
+from openstack_governance import governance
+from openstack_governance import yamlutils
+
+
+_team_data_yaml = """
+Release Management:
+  ptl:
+    name: Doug Hellmann
+    irc: dhellmann
+    email: doug@doughellmann.com
+  irc-channel: openstack-release
+  mission: >
+    Coordinating the release of OpenStack deliverables, by defining the
+    overall development cycle, release models, publication processes,
+    versioning rules and tools, then enabling project teams to produce
+    their own releases.
+  url: https://wiki.openstack.org/wiki/Release_Management
+  tags:
+    - team:diverse-affiliation
+  deliverables:
+    release-schedule-generator:
+      repos:
+        - openstack/release-schedule-generator
+    release-test:
+      repos:
+        - openstack/release-test
+    release-tools:
+      repos:
+        - openstack-infra/release-tools
+    releases:
+      repos:
+        - openstack/releases
+    reno:
+      repos:
+        - openstack/reno
+      docs:
+        contributor: https://docs.openstack.org/developer/reno/
+    specs-cookiecutter:
+      repos:
+        - openstack-dev/specs-cookiecutter
+"""
+
+TEAM_DATA = yamlutils.loads(_team_data_yaml)
+
+
+class TestGetRepoOwner(base.BaseTestCase):
+
+    def test_repo_exists(self):
+        owner = governance.get_repo_owner(
+            TEAM_DATA,
+            'openstack/releases',
+        )
+        self.assertEqual('Release Management', owner)
+
+    def test_no_such_repo(self):
+        self.assertRaises(
+            ValueError,
+            governance.get_repo_owner,
+            TEAM_DATA,
+            'openstack/no-such-repo',
+        )
+
+
+class TestGetRepositories(base.BaseTestCase):
+
+    def test_by_team(self):
+        repos = governance.get_repositories(
+            TEAM_DATA,
+            team_name='Release Management',
+        )
+        self.assertEqual(
+            sorted(['openstack/release-schedule-generator',
+                    'openstack/release-test',
+                    'openstack-infra/release-tools',
+                    'openstack/releases',
+                    'openstack/reno',
+                    'openstack-dev/specs-cookiecutter']),
+            sorted(r.name for r in repos),
+        )
+
+    def test_by_deliverable(self):
+        repos = governance.get_repositories(
+            TEAM_DATA,
+            deliverable_name='release-tools',
+        )
+        self.assertEqual(
+            ['openstack-infra/release-tools'],
+            [r.name for r in repos],
+        )
+
+    def test_tag_negative(self):
+        repos = governance.get_repositories(
+            TEAM_DATA,
+            tags=['team:single-vendor'],
+        )
+        self.assertEqual([], list(repos))
+
+    def test_tags_positive(self):
+        repos = governance.get_repositories(
+            TEAM_DATA,
+            tags=['team:diverse-affiliation'],
+        )
+        self.assertEqual(
+            sorted(['openstack/release-schedule-generator',
+                    'openstack/release-test',
+                    'openstack-infra/release-tools',
+                    'openstack/releases',
+                    'openstack/reno',
+                    'openstack-dev/specs-cookiecutter']),
+            sorted(r.name for r in repos),
+        )

openstack_governance/yamlutils.py

@@ -0,0 +1,24 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import yaml
+import yamlordereddictloader
+
+
+def loads(blob):
+    """Load a yaml blob and retain key ordering."""
+    # This does use load, which is unsafe, but should be ok
+    # for what we are loading here in this program; we should
+    # be able to fix that in the future (if it matters).
+    return yaml.load(blob, Loader=yamlordereddictloader.Loader)

test-requirements.txt

@@ -8,3 +8,15 @@ requests-cache
 Pillow>=2.4.0 # PIL License
 hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
 whereto>=0.3.0  # Apache-2.0
+
+fixtures>=3.0.0 # Apache-2.0/BSD
+python-subunit>=0.0.18 # Apache-2.0/BSD
+stestr>=2.0.0 # Apache-2.0
+testscenarios>=0.4 # Apache-2.0/BSD
+testtools>=1.4.0 # MIT
+oslotest>=1.10.0 # Apache-2.0
+
+coverage>=4.0 # Apache-2.0
+
+# mocking framework
+mock>=2.0 # BSD

tox.ini

@@ -6,8 +6,22 @@ skipsdist = True
 [testenv]
 usedevelop = True
 install_command = pip install -U {opts} {packages}
-setenv = VIRTUAL_ENV={envdir}
-deps = -r{toxinidir}/test-requirements.txt
+setenv =
+   VIRTUAL_ENV={envdir}
+   PYTHON=coverage run --source openstack_governance --parallel-mode
+   OS_STDOUT_CAPTURE=1
+   OS_STDERR_CAPTURE=1
+   OS_DEBUG=1
+   OS_LOG_CAPTURE=1
+deps =
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/test-requirements.txt
+commands =
+    stestr run '{posargs}'
+    coverage combine
+    coverage html -d cover
+    coverage xml -o cover/coverage.xml
+    coverage report --show-missing
 
 [testenv:linters]
 basepython = python3