Browse Source

Define maintain-github-openstack-mirror job

Opendev no longer automatically creates repositories on the
GitHub mirror, nor does it update descriptions or closes open PRs.

Add a playbook and a job for periodically maintaining the GitHub
mirror for the 'openstack' organization:

- updating descriptions based on Gerrit project descriptions
- creating on GitHub newly-added openstack repositories
- archiving from GitHub recently-retired openstack repositories
- closing any open PR with a healpful message

This job makes use of a GitHub API token (from the openstack-mirroring
user) and is defined to run periodically on project-config.

Change-Id: Ic02f436eb655dcbe84824b304ea2933e16312e67
changes/28/738228/5
Thierry Carrez 2 years ago
parent
commit
0197d9f92a
  1. 273
      playbooks/maintain-github-mirror/github_manager.py
  2. 29
      playbooks/maintain-github-mirror/run.yaml
  3. 18
      zuul.d/jobs.yaml
  4. 1
      zuul.d/projects.yaml
  5. 11
      zuul.d/secrets.yaml

273
playbooks/maintain-github-mirror/github_manager.py

@ -0,0 +1,273 @@
#!/usr/bin/python3
#
# Copyright 2020 Thierry Carrez <thierry@openstack.org>
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import os
import sys
import time
import github
import requests
from requests.packages import urllib3
import yaml
# Turn of warnings about bad SSL config.
# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
urllib3.disable_warnings()
DESC_SUFFIX = ' Mirror of code maintained at opendev.org.'
ARCHIVE_ORG = 'openstack-archive'
DEFAULT_TEAM_ID = 73197
PR_CLOSING_TEXT = (
"Thank you for your contribution!\n\n"
"This GitHub repository is just a mirror of %(url)s, where development "
"really happens. Pull requests proposed on GitHub are automatically "
"closed.\n\n"
"If you are interested in pushing this code upstream, please note "
"that OpenStack development uses Gerrit for change proposal and code "
"review.\n\n"
"If you have never contributed to OpenStack before, please see:\n"
"https://docs.opendev.org/opendev/infra-manual/latest/gettingstarted.html"
"\n\nFeel free to reach out on the #openstack-upstream-institute channel "
"on Freenode IRC in case you need help. You can access it through:\n"
"https://webchat.freenode.net/#openstack-upstream-institute\n"
)
def homepage(repo):
return 'https://opendev.org/' + repo
def reponame(org, fullname):
owner, name = fullname.split('/')
if org.login != owner:
raise ValueError(f'{fullname} does not match target org ({org.login})')
return name
def load_from_project_config(project_config, org='openstack'):
print('Loading project config...')
pc = {}
proj_yaml_filename = os.path.join(project_config, 'gerrit/projects.yaml')
with open(proj_yaml_filename, 'r') as proj_yaml:
for project in yaml.safe_load(proj_yaml):
if project['project'].startswith(org + '/'):
desc = project.get('description', '')
if desc and desc[-1:] != '.':
desc += '.'
desc += DESC_SUFFIX
pc[project['project']] = desc
print(f'\rLoaded {len(pc)} project descriptions from configuration.')
return pc
def list_repos_in_zuul(project_config, tenant='openstack', org='openstack'):
print('Loading Zuul repos...')
repos = set()
main_yaml_filename = os.path.join(project_config, 'zuul/main.yaml')
with open(main_yaml_filename, 'r') as main_yaml:
for t in yaml.safe_load(main_yaml):
if t['tenant']['name'] == tenant:
for ps, pt in t['tenant']['source']['gerrit'].items():
for elem in pt:
if type(elem) is dict:
for k, v in elem.items():
if k.startswith(org + '/'):
repos.add(k)
else:
if elem.startswith(org + '/'):
repos.add(elem)
print(f'Loaded {len(repos)} repositories from Gerrit.')
return repos
def list_repos_in_governance(governance, org='openstack'):
print('Loading governance repos...')
repos = set()
proj_yaml_filename = os.path.join(governance, 'reference/projects.yaml')
with open(proj_yaml_filename, 'r') as proj_yaml:
for pname, project in yaml.safe_load(proj_yaml).items():
for dname, deliv in project.get('deliverables', dict()).items():
for r in deliv['repos']:
if r.startswith(org + '/'):
repos.add(r)
extrafiles = ['sigs-repos.yaml',
'foundation-board-repos.yaml',
'technical-committee-repos.yaml',
'user-committee-repos.yaml']
for extrafile in extrafiles:
yaml_filename = os.path.join(governance, 'reference', extrafile)
with open(yaml_filename, 'r') as extra_yaml:
for pname, project in yaml.safe_load(extra_yaml).items():
for r in project:
if r['repo'].startswith(org + '/'):
repos.add(r['repo'])
print(f'Loaded {len(repos)} repositories from governance.')
return repos
def load_github_repositories(org):
print('Loading GitHub repository list from GitHub...')
gh_repos = {}
for repo in org.get_repos():
gh_repos[repo.full_name] = repo
print(f'Loaded {len(gh_repos)} repositories from GitHub. ')
return gh_repos
def transfer_repository(repofullname, newowner=ARCHIVE_ORG):
# Transfer repository is not yet supported by PyGitHub, so call directly
token = os.environ['GITHUB_TOKEN']
url = f'https://api.github.com/repos/{repofullname}/transfer'
res = requests.post(url,
headers={'Authorization': f'token {token}'},
json={'new_owner': newowner})
if res.status_code != 202:
raise github.GithubException(res.status_code, res.text)
def archive_repository(archive_org, shortname):
repository = archive_org.get_repo(shortname)
repository.edit(archived=True)
def create_repository(org, repofullname, desc, homepage):
name = reponame(org, repofullname)
org.create_repo(name=name,
description=desc,
homepage=homepage,
has_issues=False,
has_projects=False,
has_wiki=False,
team_id=DEFAULT_TEAM_ID)
def update_repository(org, repofullname, desc, homepage):
name = reponame(org, repofullname)
repository = org.get_repo(name)
repository.edit(description=desc, homepage=homepage)
def close_pull_request(org, repofullname, github_pr):
github_pr.create_issue_comment(
PR_CLOSING_TEXT % {'url': homepage(repofullname)})
github_pr.edit(state="closed")
def main(args=sys.argv[1:]):
parser = argparse.ArgumentParser()
parser.add_argument(
'project_config',
help='directory containing the project-config repository')
parser.add_argument(
'governance',
help='directory containing the governance repository')
parser.add_argument(
'--dryrun',
default=False,
help='do not actually do anything',
action='store_true')
args = parser.parse_args(args)
try:
gh = github.Github(os.environ['GITHUB_TOKEN'])
org = gh.get_organization('openstack')
archive_org = gh.get_organization(ARCHIVE_ORG)
except KeyError:
print('Aborting: missing GITHUB_TOKEN environment variable')
sys.exit(1)
if args.dryrun:
print('Running in dry run mode, no action will be actually taken')
# Load data from Gerrit and GitHub
gerrit_descriptions = load_from_project_config(args.project_config)
in_governance = list_repos_in_governance(args.governance)
in_zuul = list_repos_in_zuul(args.project_config)
if in_governance != in_zuul:
print("\nWarning, projects defined in Zuul do not match governance:")
print("\nIn Governance but not in Zuul (should be cleaned up):")
for repo in in_governance.difference(in_zuul):
print(repo)
print("\nIn Zuul but not in Governance (retirement in progress?):")
for repo in in_zuul.difference(in_governance):
print(repo)
print()
github_repos = load_github_repositories(org)
in_github = set(github_repos.keys())
print("\nUpdating repository descriptions:")
for repo in in_github.intersection(in_zuul):
if ((github_repos[repo].description != gerrit_descriptions[repo])
or (github_repos[repo].homepage != homepage(repo))):
print(repo, end=' - ', flush=True)
if args.dryrun:
print('nothing done (--dryrun)')
else:
update_repository(org, repo,
gerrit_descriptions[repo],
homepage(repo))
print('description updated')
time.sleep(1)
print("\nArchiving repositories that are in GitHub but not in Zuul:")
for repo in in_github.difference(in_zuul):
print(repo, end=' - ', flush=True)
if args.dryrun:
print('nothing done (--dryrun)')
else:
transfer_repository(repo)
print(f'moved to {ARCHIVE_ORG}', end=' - ', flush=True)
time.sleep(10)
archive_repository(archive_org, reponame(org, repo))
print('archived')
time.sleep(1)
print("\nCreating repos that are in Zuul & governance but not in Github:")
for repo in (in_zuul.intersection(in_governance)).difference(in_github):
print(repo, end=' - ', flush=True)
if args.dryrun:
print('nothing done (--dryrun)')
else:
create_repository(org, repo,
gerrit_descriptions[repo],
homepage(repo))
print('created')
time.sleep(1)
print("\nIterating through all Github repositories to close PRs:")
for repo, gh_repository in github_repos.items():
for req in gh_repository.get_pulls("open"):
print(f'Closing PR#{req.number} in {repo}', end=' - ', flush=True)
if args.dryrun:
print('nothing done (--dryrun)')
else:
close_pull_request(org, repo, req)
print('closed')
time.sleep(1)
print("Done.")
if __name__ == '__main__':
main()

29
playbooks/maintain-github-mirror/run.yaml

@ -0,0 +1,29 @@
- hosts: all
tasks:
- name: Ensure pip
include_role:
name: ensure-pip
- name: Copy manager script
copy:
src: github_manager.py
dest: "{{ ansible_user_dir }}"
- name: Install dependencies
pip:
name: github
virtualenv: "{{ ansible_user_dir }}/.venv"
virtualenv_command: "{{ ensure_pip_virtualenv_command }}"
- name: Run manager script
command: "{{ ansible_user_dir }}/.venv/bin/python {{ ansible_user_dir }}/github_manager.py {{ conf }} {{ gov }}"
environment:
GITHUB_TOKEN: "{{ github_credentials.api_token }}"
- name: Clean up after ourselves
file:
path: "{{ item }}"
state: absent
with_items:
- github_manager.py
- .venv

18
zuul.d/jobs.yaml

@ -788,6 +788,24 @@
secret: openstack-github-mirroring
pass-to-parent: true
- job:
name: maintain-github-openstack-mirror
description: |
Runs maintenance scripts for the OpenStack GitHub mirror. The script
updates the descriptions, creates new repositories, archives retired
ones, and closes any open PR. It is meant to be run periodically.
final: true
protected: true
run: playbooks/maintain-github-mirror/run.yaml
required-projects:
- name: openstack/governance
secrets:
- name: github_credentials
secret: openstack-github-mirroring
vars:
conf: "{{ zuul.projects['opendev.org/opendev/project_config'].src_dir }}"
gov: "{{ zuul.projects['opendev.org/opendev/governance'].src_dir }}"
- job:
name: propose-updates
pre-run: playbooks/proposal/pre.yaml

1
zuul.d/projects.yaml

@ -4579,6 +4579,7 @@
periodic:
jobs:
- propose-project-config-update
- maintain-github-openstack-mirror
opendev-prod-hourly:
jobs:
- publish-irc-meetings

11
zuul.d/secrets.yaml

@ -773,3 +773,14 @@
xRCWUjMcQUQFKqUB4e3VBY9mOxrqACesPuE5H0avVwIwaPGZngPgDBmDyZDM1X28N9SvZ
h0x++Jcyt5MX2HULKvnmhMeRuskGrr/4ujaJz+X85j/qAySY0T+euU5QoDaAjRNNQeoaO
X3PXCRgfNXahgWYgq1Xu5l9wLhXaEpWQuOj6BGVv/Fy2yVR89b/1PryYzRCpoI=
api_token: !encrypted/pkcs1-oaep
- P4WWrBQZeA/KpKrqjBpU8j6Htql+J3kHbauaQqZVoEkcggj214O45TNSpz40ZqOGZNdbj
38/tp7ofTkfUNTMYR27/fngZgHXFTLm9SvLb0lvoML5QthtaWY3UCTkBAlQ5P/XffIFff
p3zvYzh8SI+8hFrgr5Wod7xvqBTeQOgsR2pMHb2EJRy29Ru0ZD4DqF2YICdjQsAt9QGYn
deD7u5IWfgfeh3CdtYRX9s2NPpBLM1Fz57Nd4ssptjIhOLBw/Q+UhK2W2ZB9fpc7AKVvs
EfZObOuU8vzEzhEoRSJrBsjaImi9bwZZmucV4BupZVEaVprOWr7AHgEZZ3R1AtBOFt3D/
uhKNO6IstRKioxyqm96k+cSOCVbEhpm/ibktgJR8Im6KiRGez2MENEuGDk0Wu//xyF6/O
q28o1lSJIop8n1dxFAmjEaxyarl/GqUWQEh2XErhvDVsT5IRtav0yvihb3v/495SP7qNm
HETs35aycMm/KTjhryPoQbsAAmVe/i/+PFcyxcMDPZHQmJcWRD+K7lJb3o18kNST410B/
FtiR0LGtDxKM1bdi57Qc3f43P4jzY3Px07SSKVFKSkuI1zSLnsZSbmWg/wBHcjllsA73L
l0HItpoMi3S3KDsFajJbk2UE6NhCBD7kmsSB69L6yb7VJdKZqMAHS2BSSXIRdA=

Loading…
Cancel
Save