From 5a4e76328b4230777bc8cc64334379a88e7536a0 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 12 Oct 2018 13:20:19 -0400 Subject: [PATCH] add module for fetching and parsing governance data Change-Id: I7279023e5d81554e19f315fe12ab6fdb988af934 Signed-off-by: Doug Hellmann --- .gitignore | 6 +- .stestr.conf | 3 + .zuul.yaml | 1 + openstack_governance/governance.py | 149 ++++++++++++++++++ openstack_governance/tests/__init__.py | 0 openstack_governance/tests/test_governance.py | 127 +++++++++++++++ openstack_governance/yamlutils.py | 24 +++ test-requirements.txt | 12 ++ tox.ini | 18 ++- 9 files changed, 337 insertions(+), 3 deletions(-) create mode 100644 .stestr.conf create mode 100644 openstack_governance/governance.py create mode 100644 openstack_governance/tests/__init__.py create mode 100644 openstack_governance/tests/test_governance.py create mode 100644 openstack_governance/yamlutils.py diff --git a/.gitignore b/.gitignore index 3515f25fb..29a129d65 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,8 @@ pbr*.egg reference/projects/*.rst *.pyc AUTHORS -ChangeLog \ No newline at end of file +ChangeLog +.stestr +.coverage.* +.coverage +cover/ \ No newline at end of file diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 000000000..fac76efa7 --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=./openstack_governance/tests +top_dir=./ diff --git a/.zuul.yaml b/.zuul.yaml index 1fdf9de47..40d4c9046 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,6 +1,7 @@ - project: templates: - build-openstack-docs-pti + - openstack-python36-jobs check: jobs: - openstack-tox-linters: diff --git a/openstack_governance/governance.py b/openstack_governance/governance.py new file mode 100644 index 000000000..def64928a --- /dev/null +++ b/openstack_governance/governance.py @@ -0,0 +1,149 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Work with the governance repository. +""" + +import weakref + +from openstack_governance import yamlutils + +import requests + +PROJECTS_LIST = "http://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml" # noqa + + +def get_team_data(url=PROJECTS_LIST): + """Return the parsed team data from the governance repository. + + :param url: Optional URL to the location of the projects.yaml + file. Defaults to the most current version in the public git + repository. + + """ + r = requests.get(url) + return yamlutils.loads(r.text) + + +def get_tags_for_deliverable(team_data, team, name): + "Return the tags for the deliverable owned by the team." + if team not in team_data: + return set() + team_info = team_data[team] + dinfo = team_info['deliverables'].get(name) + if not dinfo: + return set() + return set(dinfo.get('tags', [])).union(set(team_info.get('tags', []))) + + +def get_repo_owner(team_data, repo_name): + """Return the name of the team that owns the repository. + + :param team_data: The result of calling :func:`get_team_data` + :param repo_name: Long name of the repository, such as 'openstack/nova'. + + """ + for team, info in team_data.items(): + for dname, dinfo in info.get('deliverables', {}).items(): + if repo_name in dinfo.get('repos', []): + return team + raise ValueError('Repository %s not found in governance list' % repo_name) + + +class Team(object): + _liaison_data = None + + def __init__(self, name, data): + self.name = name + self.data = data + # Protectively initialize the ptl data structure in case part + # of it is missing from the project list, then replace any + # values we do have from that data. + self.ptl = { + 'name': 'MISSING', + 'irc': 'MISSING', + } + self.ptl.update(data.get('ptl', {})) + self.deliverables = { + dn: Deliverable(dn, di, self) + for dn, di in self.data.get('deliverables', {}).items() + } + + @property + def tags(self): + return set(self.data.get('tags', [])) + + +class Deliverable(object): + def __init__(self, name, data, team): + self.name = name + self.data = data + self.team = weakref.proxy(team) + self.repositories = { + rn: Repository(rn, self) + for rn in self.data.get('repos', []) + } + + @property + def tags(self): + return set(self.data.get('tags', [])).union(self.team.tags) + + +class Repository(object): + def __init__(self, name, deliverable): + self.name = name + self.deliverable = weakref.proxy(deliverable) + + @property + def tags(self): + return self.deliverable.tags + + +def get_repositories(team_data, team_name=None, deliverable_name=None, + tags=[], code_only=False): + """Return a sequence of repositories, possibly filtered. + + :param team_data: The result of calling :func:`get_team_data` + :param team_name: The name of the team owning the repositories. Can be + None. + :para deliverable_name: The name of the deliverable to which all + repos should belong. + :param tags: The names of any tags the repositories should + have. Can be empty. + :param code_only: Boolean indicating whether to return only code + repositories (ignoring specs and cookiecutter templates). + + """ + if tags: + tags = set(tags) + if team_name: + try: + teams = [Team(team_name, team_data[team_name])] + except KeyError: + raise RuntimeError('No team %r found in %r' % + (team_name, list(team_data.keys()))) + else: + teams = [Team(n, i) for n, i in team_data.items()] + for team in teams: + if deliverable_name and deliverable_name not in team.deliverables: + continue + if deliverable_name: + deliverables = [team.deliverables[deliverable_name]] + else: + deliverables = team.deliverables.values() + for deliverable in deliverables: + for repository in deliverable.repositories.values(): + if tags and not tags.issubset(repository.tags): + continue + if code_only and not repository.code_related: + continue + yield repository diff --git a/openstack_governance/tests/__init__.py b/openstack_governance/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openstack_governance/tests/test_governance.py b/openstack_governance/tests/test_governance.py new file mode 100644 index 000000000..0d8de8935 --- /dev/null +++ b/openstack_governance/tests/test_governance.py @@ -0,0 +1,127 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslotest import base + +from openstack_governance import governance +from openstack_governance import yamlutils + + +_team_data_yaml = """ +Release Management: + ptl: + name: Doug Hellmann + irc: dhellmann + email: doug@doughellmann.com + irc-channel: openstack-release + mission: > + Coordinating the release of OpenStack deliverables, by defining the + overall development cycle, release models, publication processes, + versioning rules and tools, then enabling project teams to produce + their own releases. + url: https://wiki.openstack.org/wiki/Release_Management + tags: + - team:diverse-affiliation + deliverables: + release-schedule-generator: + repos: + - openstack/release-schedule-generator + release-test: + repos: + - openstack/release-test + release-tools: + repos: + - openstack-infra/release-tools + releases: + repos: + - openstack/releases + reno: + repos: + - openstack/reno + docs: + contributor: https://docs.openstack.org/developer/reno/ + specs-cookiecutter: + repos: + - openstack-dev/specs-cookiecutter +""" + +TEAM_DATA = yamlutils.loads(_team_data_yaml) + + +class TestGetRepoOwner(base.BaseTestCase): + + def test_repo_exists(self): + owner = governance.get_repo_owner( + TEAM_DATA, + 'openstack/releases', + ) + self.assertEqual('Release Management', owner) + + def test_no_such_repo(self): + self.assertRaises( + ValueError, + governance.get_repo_owner, + TEAM_DATA, + 'openstack/no-such-repo', + ) + + +class TestGetRepositories(base.BaseTestCase): + + def test_by_team(self): + repos = governance.get_repositories( + TEAM_DATA, + team_name='Release Management', + ) + self.assertEqual( + sorted(['openstack/release-schedule-generator', + 'openstack/release-test', + 'openstack-infra/release-tools', + 'openstack/releases', + 'openstack/reno', + 'openstack-dev/specs-cookiecutter']), + sorted(r.name for r in repos), + ) + + def test_by_deliverable(self): + repos = governance.get_repositories( + TEAM_DATA, + deliverable_name='release-tools', + ) + self.assertEqual( + ['openstack-infra/release-tools'], + [r.name for r in repos], + ) + + def test_tag_negative(self): + repos = governance.get_repositories( + TEAM_DATA, + tags=['team:single-vendor'], + ) + self.assertEqual([], list(repos)) + + def test_tags_positive(self): + repos = governance.get_repositories( + TEAM_DATA, + tags=['team:diverse-affiliation'], + ) + self.assertEqual( + sorted(['openstack/release-schedule-generator', + 'openstack/release-test', + 'openstack-infra/release-tools', + 'openstack/releases', + 'openstack/reno', + 'openstack-dev/specs-cookiecutter']), + sorted(r.name for r in repos), + ) diff --git a/openstack_governance/yamlutils.py b/openstack_governance/yamlutils.py new file mode 100644 index 000000000..aafac728f --- /dev/null +++ b/openstack_governance/yamlutils.py @@ -0,0 +1,24 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import yaml +import yamlordereddictloader + + +def loads(blob): + """Load a yaml blob and retain key ordering.""" + # This does use load, which is unsafe, but should be ok + # for what we are loading here in this program; we should + # be able to fix that in the future (if it matters). + return yaml.load(blob, Loader=yamlordereddictloader.Loader) diff --git a/test-requirements.txt b/test-requirements.txt index bd1a05663..969944637 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,3 +8,15 @@ requests-cache Pillow>=2.4.0 # PIL License hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 whereto>=0.3.0 # Apache-2.0 + +fixtures>=3.0.0 # Apache-2.0/BSD +python-subunit>=0.0.18 # Apache-2.0/BSD +stestr>=2.0.0 # Apache-2.0 +testscenarios>=0.4 # Apache-2.0/BSD +testtools>=1.4.0 # MIT +oslotest>=1.10.0 # Apache-2.0 + +coverage>=4.0 # Apache-2.0 + +# mocking framework +mock>=2.0 # BSD diff --git a/tox.ini b/tox.ini index d001ef17f..c9d365f0b 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,22 @@ skipsdist = True [testenv] usedevelop = True install_command = pip install -U {opts} {packages} -setenv = VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/test-requirements.txt +setenv = + VIRTUAL_ENV={envdir} + PYTHON=coverage run --source openstack_governance --parallel-mode + OS_STDOUT_CAPTURE=1 + OS_STDERR_CAPTURE=1 + OS_DEBUG=1 + OS_LOG_CAPTURE=1 +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + stestr run '{posargs}' + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report --show-missing [testenv:linters] basepython = python3