add module for fetching and parsing governance data

Change-Id: I7279023e5d81554e19f315fe12ab6fdb988af934
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2018-10-12 13:20:19 -04:00
parent 381f7b2d14
commit 5a4e76328b
9 changed files with 337 additions and 3 deletions

4
.gitignore vendored
View File

@ -6,3 +6,7 @@ reference/projects/*.rst
*.pyc
AUTHORS
ChangeLog
.stestr
.coverage.*
.coverage
cover/

3
.stestr.conf Normal file
View File

@ -0,0 +1,3 @@
[DEFAULT]
test_path=./openstack_governance/tests
top_dir=./

View File

@ -1,6 +1,7 @@
- project:
templates:
- build-openstack-docs-pti
- openstack-python36-jobs
check:
jobs:
- openstack-tox-linters:

View File

@ -0,0 +1,149 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Work with the governance repository.
"""
import weakref
from openstack_governance import yamlutils
import requests
PROJECTS_LIST = "http://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml" # noqa
def get_team_data(url=PROJECTS_LIST):
"""Return the parsed team data from the governance repository.
:param url: Optional URL to the location of the projects.yaml
file. Defaults to the most current version in the public git
repository.
"""
r = requests.get(url)
return yamlutils.loads(r.text)
def get_tags_for_deliverable(team_data, team, name):
"Return the tags for the deliverable owned by the team."
if team not in team_data:
return set()
team_info = team_data[team]
dinfo = team_info['deliverables'].get(name)
if not dinfo:
return set()
return set(dinfo.get('tags', [])).union(set(team_info.get('tags', [])))
def get_repo_owner(team_data, repo_name):
"""Return the name of the team that owns the repository.
:param team_data: The result of calling :func:`get_team_data`
:param repo_name: Long name of the repository, such as 'openstack/nova'.
"""
for team, info in team_data.items():
for dname, dinfo in info.get('deliverables', {}).items():
if repo_name in dinfo.get('repos', []):
return team
raise ValueError('Repository %s not found in governance list' % repo_name)
class Team(object):
_liaison_data = None
def __init__(self, name, data):
self.name = name
self.data = data
# Protectively initialize the ptl data structure in case part
# of it is missing from the project list, then replace any
# values we do have from that data.
self.ptl = {
'name': 'MISSING',
'irc': 'MISSING',
}
self.ptl.update(data.get('ptl', {}))
self.deliverables = {
dn: Deliverable(dn, di, self)
for dn, di in self.data.get('deliverables', {}).items()
}
@property
def tags(self):
return set(self.data.get('tags', []))
class Deliverable(object):
def __init__(self, name, data, team):
self.name = name
self.data = data
self.team = weakref.proxy(team)
self.repositories = {
rn: Repository(rn, self)
for rn in self.data.get('repos', [])
}
@property
def tags(self):
return set(self.data.get('tags', [])).union(self.team.tags)
class Repository(object):
def __init__(self, name, deliverable):
self.name = name
self.deliverable = weakref.proxy(deliverable)
@property
def tags(self):
return self.deliverable.tags
def get_repositories(team_data, team_name=None, deliverable_name=None,
tags=[], code_only=False):
"""Return a sequence of repositories, possibly filtered.
:param team_data: The result of calling :func:`get_team_data`
:param team_name: The name of the team owning the repositories. Can be
None.
:para deliverable_name: The name of the deliverable to which all
repos should belong.
:param tags: The names of any tags the repositories should
have. Can be empty.
:param code_only: Boolean indicating whether to return only code
repositories (ignoring specs and cookiecutter templates).
"""
if tags:
tags = set(tags)
if team_name:
try:
teams = [Team(team_name, team_data[team_name])]
except KeyError:
raise RuntimeError('No team %r found in %r' %
(team_name, list(team_data.keys())))
else:
teams = [Team(n, i) for n, i in team_data.items()]
for team in teams:
if deliverable_name and deliverable_name not in team.deliverables:
continue
if deliverable_name:
deliverables = [team.deliverables[deliverable_name]]
else:
deliverables = team.deliverables.values()
for deliverable in deliverables:
for repository in deliverable.repositories.values():
if tags and not tags.issubset(repository.tags):
continue
if code_only and not repository.code_related:
continue
yield repository

View File

View File

@ -0,0 +1,127 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslotest import base
from openstack_governance import governance
from openstack_governance import yamlutils
_team_data_yaml = """
Release Management:
ptl:
name: Doug Hellmann
irc: dhellmann
email: doug@doughellmann.com
irc-channel: openstack-release
mission: >
Coordinating the release of OpenStack deliverables, by defining the
overall development cycle, release models, publication processes,
versioning rules and tools, then enabling project teams to produce
their own releases.
url: https://wiki.openstack.org/wiki/Release_Management
tags:
- team:diverse-affiliation
deliverables:
release-schedule-generator:
repos:
- openstack/release-schedule-generator
release-test:
repos:
- openstack/release-test
release-tools:
repos:
- openstack-infra/release-tools
releases:
repos:
- openstack/releases
reno:
repos:
- openstack/reno
docs:
contributor: https://docs.openstack.org/developer/reno/
specs-cookiecutter:
repos:
- openstack-dev/specs-cookiecutter
"""
TEAM_DATA = yamlutils.loads(_team_data_yaml)
class TestGetRepoOwner(base.BaseTestCase):
def test_repo_exists(self):
owner = governance.get_repo_owner(
TEAM_DATA,
'openstack/releases',
)
self.assertEqual('Release Management', owner)
def test_no_such_repo(self):
self.assertRaises(
ValueError,
governance.get_repo_owner,
TEAM_DATA,
'openstack/no-such-repo',
)
class TestGetRepositories(base.BaseTestCase):
def test_by_team(self):
repos = governance.get_repositories(
TEAM_DATA,
team_name='Release Management',
)
self.assertEqual(
sorted(['openstack/release-schedule-generator',
'openstack/release-test',
'openstack-infra/release-tools',
'openstack/releases',
'openstack/reno',
'openstack-dev/specs-cookiecutter']),
sorted(r.name for r in repos),
)
def test_by_deliverable(self):
repos = governance.get_repositories(
TEAM_DATA,
deliverable_name='release-tools',
)
self.assertEqual(
['openstack-infra/release-tools'],
[r.name for r in repos],
)
def test_tag_negative(self):
repos = governance.get_repositories(
TEAM_DATA,
tags=['team:single-vendor'],
)
self.assertEqual([], list(repos))
def test_tags_positive(self):
repos = governance.get_repositories(
TEAM_DATA,
tags=['team:diverse-affiliation'],
)
self.assertEqual(
sorted(['openstack/release-schedule-generator',
'openstack/release-test',
'openstack-infra/release-tools',
'openstack/releases',
'openstack/reno',
'openstack-dev/specs-cookiecutter']),
sorted(r.name for r in repos),
)

View File

@ -0,0 +1,24 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaml
import yamlordereddictloader
def loads(blob):
"""Load a yaml blob and retain key ordering."""
# This does use load, which is unsafe, but should be ok
# for what we are loading here in this program; we should
# be able to fix that in the future (if it matters).
return yaml.load(blob, Loader=yamlordereddictloader.Loader)

View File

@ -8,3 +8,15 @@ requests-cache
Pillow>=2.4.0 # PIL License
hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
whereto>=0.3.0 # Apache-2.0
fixtures>=3.0.0 # Apache-2.0/BSD
python-subunit>=0.0.18 # Apache-2.0/BSD
stestr>=2.0.0 # Apache-2.0
testscenarios>=0.4 # Apache-2.0/BSD
testtools>=1.4.0 # MIT
oslotest>=1.10.0 # Apache-2.0
coverage>=4.0 # Apache-2.0
# mocking framework
mock>=2.0 # BSD

18
tox.ini
View File

@ -6,8 +6,22 @@ skipsdist = True
[testenv]
usedevelop = True
install_command = pip install -U {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/test-requirements.txt
setenv =
VIRTUAL_ENV={envdir}
PYTHON=coverage run --source openstack_governance --parallel-mode
OS_STDOUT_CAPTURE=1
OS_STDERR_CAPTURE=1
OS_DEBUG=1
OS_LOG_CAPTURE=1
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands =
stestr run '{posargs}'
coverage combine
coverage html -d cover
coverage xml -o cover/coverage.xml
coverage report --show-missing
[testenv:linters]
basepython = python3