From 0197d9f92ab7c2b40206b8e8054f34bff9f36d7e Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Thu, 25 Jun 2020 14:56:24 +0200 Subject: [PATCH] Define maintain-github-openstack-mirror job Opendev no longer automatically creates repositories on the GitHub mirror, nor does it update descriptions or closes open PRs. Add a playbook and a job for periodically maintaining the GitHub mirror for the 'openstack' organization: - updating descriptions based on Gerrit project descriptions - creating on GitHub newly-added openstack repositories - archiving from GitHub recently-retired openstack repositories - closing any open PR with a healpful message This job makes use of a GitHub API token (from the openstack-mirroring user) and is defined to run periodically on project-config. Change-Id: Ic02f436eb655dcbe84824b304ea2933e16312e67 --- .../maintain-github-mirror/github_manager.py | 273 ++++++++++++++++++ playbooks/maintain-github-mirror/run.yaml | 29 ++ zuul.d/jobs.yaml | 18 ++ zuul.d/projects.yaml | 1 + zuul.d/secrets.yaml | 11 + 5 files changed, 332 insertions(+) create mode 100755 playbooks/maintain-github-mirror/github_manager.py create mode 100644 playbooks/maintain-github-mirror/run.yaml diff --git a/playbooks/maintain-github-mirror/github_manager.py b/playbooks/maintain-github-mirror/github_manager.py new file mode 100755 index 0000000000..27f49207b7 --- /dev/null +++ b/playbooks/maintain-github-mirror/github_manager.py @@ -0,0 +1,273 @@ +#!/usr/bin/python3 +# +# Copyright 2020 Thierry Carrez +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os +import sys +import time + +import github +import requests +from requests.packages import urllib3 +import yaml + + +# Turn of warnings about bad SSL config. +# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings +urllib3.disable_warnings() + +DESC_SUFFIX = ' Mirror of code maintained at opendev.org.' +ARCHIVE_ORG = 'openstack-archive' +DEFAULT_TEAM_ID = 73197 +PR_CLOSING_TEXT = ( + "Thank you for your contribution!\n\n" + "This GitHub repository is just a mirror of %(url)s, where development " + "really happens. Pull requests proposed on GitHub are automatically " + "closed.\n\n" + "If you are interested in pushing this code upstream, please note " + "that OpenStack development uses Gerrit for change proposal and code " + "review.\n\n" + "If you have never contributed to OpenStack before, please see:\n" + "https://docs.opendev.org/opendev/infra-manual/latest/gettingstarted.html" + "\n\nFeel free to reach out on the #openstack-upstream-institute channel " + "on Freenode IRC in case you need help. You can access it through:\n" + "https://webchat.freenode.net/#openstack-upstream-institute\n" +) + + +def homepage(repo): + return 'https://opendev.org/' + repo + + +def reponame(org, fullname): + owner, name = fullname.split('/') + if org.login != owner: + raise ValueError(f'{fullname} does not match target org ({org.login})') + return name + + +def load_from_project_config(project_config, org='openstack'): + print('Loading project config...') + pc = {} + proj_yaml_filename = os.path.join(project_config, 'gerrit/projects.yaml') + with open(proj_yaml_filename, 'r') as proj_yaml: + for project in yaml.safe_load(proj_yaml): + if project['project'].startswith(org + '/'): + desc = project.get('description', '') + if desc and desc[-1:] != '.': + desc += '.' + desc += DESC_SUFFIX + pc[project['project']] = desc + print(f'\rLoaded {len(pc)} project descriptions from configuration.') + return pc + + +def list_repos_in_zuul(project_config, tenant='openstack', org='openstack'): + print('Loading Zuul repos...') + repos = set() + main_yaml_filename = os.path.join(project_config, 'zuul/main.yaml') + with open(main_yaml_filename, 'r') as main_yaml: + for t in yaml.safe_load(main_yaml): + if t['tenant']['name'] == tenant: + for ps, pt in t['tenant']['source']['gerrit'].items(): + for elem in pt: + if type(elem) is dict: + for k, v in elem.items(): + if k.startswith(org + '/'): + repos.add(k) + else: + if elem.startswith(org + '/'): + repos.add(elem) + print(f'Loaded {len(repos)} repositories from Gerrit.') + return repos + + +def list_repos_in_governance(governance, org='openstack'): + print('Loading governance repos...') + repos = set() + + proj_yaml_filename = os.path.join(governance, 'reference/projects.yaml') + with open(proj_yaml_filename, 'r') as proj_yaml: + for pname, project in yaml.safe_load(proj_yaml).items(): + for dname, deliv in project.get('deliverables', dict()).items(): + for r in deliv['repos']: + if r.startswith(org + '/'): + repos.add(r) + + extrafiles = ['sigs-repos.yaml', + 'foundation-board-repos.yaml', + 'technical-committee-repos.yaml', + 'user-committee-repos.yaml'] + for extrafile in extrafiles: + yaml_filename = os.path.join(governance, 'reference', extrafile) + with open(yaml_filename, 'r') as extra_yaml: + for pname, project in yaml.safe_load(extra_yaml).items(): + for r in project: + if r['repo'].startswith(org + '/'): + repos.add(r['repo']) + print(f'Loaded {len(repos)} repositories from governance.') + return repos + + +def load_github_repositories(org): + print('Loading GitHub repository list from GitHub...') + gh_repos = {} + for repo in org.get_repos(): + gh_repos[repo.full_name] = repo + print(f'Loaded {len(gh_repos)} repositories from GitHub. ') + return gh_repos + + +def transfer_repository(repofullname, newowner=ARCHIVE_ORG): + # Transfer repository is not yet supported by PyGitHub, so call directly + token = os.environ['GITHUB_TOKEN'] + url = f'https://api.github.com/repos/{repofullname}/transfer' + res = requests.post(url, + headers={'Authorization': f'token {token}'}, + json={'new_owner': newowner}) + if res.status_code != 202: + raise github.GithubException(res.status_code, res.text) + + +def archive_repository(archive_org, shortname): + repository = archive_org.get_repo(shortname) + repository.edit(archived=True) + + +def create_repository(org, repofullname, desc, homepage): + name = reponame(org, repofullname) + org.create_repo(name=name, + description=desc, + homepage=homepage, + has_issues=False, + has_projects=False, + has_wiki=False, + team_id=DEFAULT_TEAM_ID) + + +def update_repository(org, repofullname, desc, homepage): + name = reponame(org, repofullname) + repository = org.get_repo(name) + repository.edit(description=desc, homepage=homepage) + + +def close_pull_request(org, repofullname, github_pr): + github_pr.create_issue_comment( + PR_CLOSING_TEXT % {'url': homepage(repofullname)}) + github_pr.edit(state="closed") + + +def main(args=sys.argv[1:]): + parser = argparse.ArgumentParser() + parser.add_argument( + 'project_config', + help='directory containing the project-config repository') + parser.add_argument( + 'governance', + help='directory containing the governance repository') + parser.add_argument( + '--dryrun', + default=False, + help='do not actually do anything', + action='store_true') + args = parser.parse_args(args) + + try: + gh = github.Github(os.environ['GITHUB_TOKEN']) + org = gh.get_organization('openstack') + archive_org = gh.get_organization(ARCHIVE_ORG) + except KeyError: + print('Aborting: missing GITHUB_TOKEN environment variable') + sys.exit(1) + + if args.dryrun: + print('Running in dry run mode, no action will be actually taken') + + # Load data from Gerrit and GitHub + gerrit_descriptions = load_from_project_config(args.project_config) + in_governance = list_repos_in_governance(args.governance) + in_zuul = list_repos_in_zuul(args.project_config) + + if in_governance != in_zuul: + print("\nWarning, projects defined in Zuul do not match governance:") + + print("\nIn Governance but not in Zuul (should be cleaned up):") + for repo in in_governance.difference(in_zuul): + print(repo) + + print("\nIn Zuul but not in Governance (retirement in progress?):") + for repo in in_zuul.difference(in_governance): + print(repo) + print() + + github_repos = load_github_repositories(org) + in_github = set(github_repos.keys()) + + print("\nUpdating repository descriptions:") + for repo in in_github.intersection(in_zuul): + if ((github_repos[repo].description != gerrit_descriptions[repo]) + or (github_repos[repo].homepage != homepage(repo))): + print(repo, end=' - ', flush=True) + if args.dryrun: + print('nothing done (--dryrun)') + else: + update_repository(org, repo, + gerrit_descriptions[repo], + homepage(repo)) + print('description updated') + time.sleep(1) + + print("\nArchiving repositories that are in GitHub but not in Zuul:") + for repo in in_github.difference(in_zuul): + print(repo, end=' - ', flush=True) + if args.dryrun: + print('nothing done (--dryrun)') + else: + transfer_repository(repo) + print(f'moved to {ARCHIVE_ORG}', end=' - ', flush=True) + time.sleep(10) + archive_repository(archive_org, reponame(org, repo)) + print('archived') + time.sleep(1) + + print("\nCreating repos that are in Zuul & governance but not in Github:") + for repo in (in_zuul.intersection(in_governance)).difference(in_github): + print(repo, end=' - ', flush=True) + if args.dryrun: + print('nothing done (--dryrun)') + else: + create_repository(org, repo, + gerrit_descriptions[repo], + homepage(repo)) + print('created') + time.sleep(1) + + print("\nIterating through all Github repositories to close PRs:") + for repo, gh_repository in github_repos.items(): + for req in gh_repository.get_pulls("open"): + print(f'Closing PR#{req.number} in {repo}', end=' - ', flush=True) + if args.dryrun: + print('nothing done (--dryrun)') + else: + close_pull_request(org, repo, req) + print('closed') + time.sleep(1) + print("Done.") + + +if __name__ == '__main__': + main() diff --git a/playbooks/maintain-github-mirror/run.yaml b/playbooks/maintain-github-mirror/run.yaml new file mode 100644 index 0000000000..5dd67f8a6e --- /dev/null +++ b/playbooks/maintain-github-mirror/run.yaml @@ -0,0 +1,29 @@ +- hosts: all + tasks: + - name: Ensure pip + include_role: + name: ensure-pip + + - name: Copy manager script + copy: + src: github_manager.py + dest: "{{ ansible_user_dir }}" + + - name: Install dependencies + pip: + name: github + virtualenv: "{{ ansible_user_dir }}/.venv" + virtualenv_command: "{{ ensure_pip_virtualenv_command }}" + + - name: Run manager script + command: "{{ ansible_user_dir }}/.venv/bin/python {{ ansible_user_dir }}/github_manager.py {{ conf }} {{ gov }}" + environment: + GITHUB_TOKEN: "{{ github_credentials.api_token }}" + + - name: Clean up after ourselves + file: + path: "{{ item }}" + state: absent + with_items: + - github_manager.py + - .venv diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index f60cc4aa9c..c90ee915e3 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -788,6 +788,24 @@ secret: openstack-github-mirroring pass-to-parent: true +- job: + name: maintain-github-openstack-mirror + description: | + Runs maintenance scripts for the OpenStack GitHub mirror. The script + updates the descriptions, creates new repositories, archives retired + ones, and closes any open PR. It is meant to be run periodically. + final: true + protected: true + run: playbooks/maintain-github-mirror/run.yaml + required-projects: + - name: openstack/governance + secrets: + - name: github_credentials + secret: openstack-github-mirroring + vars: + conf: "{{ zuul.projects['opendev.org/opendev/project_config'].src_dir }}" + gov: "{{ zuul.projects['opendev.org/opendev/governance'].src_dir }}" + - job: name: propose-updates pre-run: playbooks/proposal/pre.yaml diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index f5f8942f16..45710e1850 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -4579,6 +4579,7 @@ periodic: jobs: - propose-project-config-update + - maintain-github-openstack-mirror opendev-prod-hourly: jobs: - publish-irc-meetings diff --git a/zuul.d/secrets.yaml b/zuul.d/secrets.yaml index b9fc688982..bf9f300136 100644 --- a/zuul.d/secrets.yaml +++ b/zuul.d/secrets.yaml @@ -773,3 +773,14 @@ xRCWUjMcQUQFKqUB4e3VBY9mOxrqACesPuE5H0avVwIwaPGZngPgDBmDyZDM1X28N9SvZ h0x++Jcyt5MX2HULKvnmhMeRuskGrr/4ujaJz+X85j/qAySY0T+euU5QoDaAjRNNQeoaO X3PXCRgfNXahgWYgq1Xu5l9wLhXaEpWQuOj6BGVv/Fy2yVR89b/1PryYzRCpoI= + api_token: !encrypted/pkcs1-oaep + - P4WWrBQZeA/KpKrqjBpU8j6Htql+J3kHbauaQqZVoEkcggj214O45TNSpz40ZqOGZNdbj + 38/tp7ofTkfUNTMYR27/fngZgHXFTLm9SvLb0lvoML5QthtaWY3UCTkBAlQ5P/XffIFff + p3zvYzh8SI+8hFrgr5Wod7xvqBTeQOgsR2pMHb2EJRy29Ru0ZD4DqF2YICdjQsAt9QGYn + deD7u5IWfgfeh3CdtYRX9s2NPpBLM1Fz57Nd4ssptjIhOLBw/Q+UhK2W2ZB9fpc7AKVvs + EfZObOuU8vzEzhEoRSJrBsjaImi9bwZZmucV4BupZVEaVprOWr7AHgEZZ3R1AtBOFt3D/ + uhKNO6IstRKioxyqm96k+cSOCVbEhpm/ibktgJR8Im6KiRGez2MENEuGDk0Wu//xyF6/O + q28o1lSJIop8n1dxFAmjEaxyarl/GqUWQEh2XErhvDVsT5IRtav0yvihb3v/495SP7qNm + HETs35aycMm/KTjhryPoQbsAAmVe/i/+PFcyxcMDPZHQmJcWRD+K7lJb3o18kNST410B/ + FtiR0LGtDxKM1bdi57Qc3f43P4jzY3Px07SSKVFKSkuI1zSLnsZSbmWg/wBHcjllsA73L + l0HItpoMi3S3KDsFajJbk2UE6NhCBD7kmsSB69L6yb7VJdKZqMAHS2BSSXIRdA=