#!/usr/bin/python3
#
# Copyright 2020 Thierry Carrez <thierry@openstack.org>
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import os
import sys
import time
import github
import requests
from requests.packages import urllib3
import yaml
# Turn of warnings about bad SSL config.
# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
urllib3.disable_warnings()
DESC_SUFFIX = ' Mirror of code maintained at opendev.org.'
ARCHIVE_ORG = 'openstack-archive'
DEFAULT_TEAM_ID = 73197
PR_CLOSING_TEXT = (
"Thank you for your contribution!\n\n"
"This GitHub repository is just a mirror of %(url)s, where development "
"really happens. Pull requests proposed on GitHub are automatically "
"closed.\n\n"
"If you are interested in pushing this code upstream, please note "
"that OpenStack development uses Gerrit for change proposal and code "
"review.\n\n"
"If you have never contributed to OpenStack before, please see:\n"
"https://docs.openstack.org/contributors/code-and-documentation/"
"quick-start.html"
"\n\nFeel free to reach out to the First Contact SIG by sending an "
"email to the openstack-discuss list with the tag '[First Contact]' "
"in the subject line. To email the mailing list, you must first "
"subscribe which can be done here:\n"
"http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-discuss"
)
def homepage(repo):
return 'https://opendev.org/' + repo
def reponame(org, fullname):
owner, name = fullname.split('/')
if org.login != owner:
raise ValueError(f'{fullname} does not match target org ({org.login})')
return name
def load_from_project_config(project_config, org='openstack'):
print('Loading project config...')
pc = {}
proj_yaml_filename = os.path.join(project_config, 'gerrit/projects.yaml')
with open(proj_yaml_filename, 'r') as proj_yaml:
for project in yaml.safe_load(proj_yaml):
if project['project'].startswith(org + '/'):
desc = project.get('description', '')
if desc and desc[-1:] != '.':
desc += '.'
desc += DESC_SUFFIX
pc[project['project']] = desc
print(f'\rLoaded {len(pc)} project descriptions from configuration.')
return pc
def list_repos_in_zuul(project_config, tenant='openstack', org='openstack'):
print('Loading Zuul repos...')
repos = set()
main_yaml_filename = os.path.join(project_config, 'zuul/main.yaml')
with open(main_yaml_filename, 'r') as main_yaml:
for t in yaml.safe_load(main_yaml):
if 'tenant' in t and t['tenant']['name'] == tenant:
for ps, pt in t['tenant']['source']['gerrit'].items():
for elem in pt:
if type(elem) is dict:
for k, v in elem.items():
if k.startswith(org + '/'):
repos.add(k)
else:
if elem.startswith(org + '/'):
repos.add(elem)
print(f'Loaded {len(repos)} repositories from Gerrit.')
return repos
def list_repos_in_governance(governance, org='openstack'):
print('Loading governance repos...')
repos = set()
proj_yaml_filename = os.path.join(governance, 'reference/projects.yaml')
with open(proj_yaml_filename, 'r') as proj_yaml:
for pname, project in yaml.safe_load(proj_yaml).items():
for dname, deliv in project.get('deliverables', dict()).items():
for r in deliv['repos']:
if r.startswith(org + '/'):
repos.add(r)
extrafiles = ['sigs-repos.yaml',
'technical-committee-repos.yaml',
'user-committee-repos.yaml']
for extrafile in extrafiles:
yaml_filename = os.path.join(governance, 'reference', extrafile)
if not os.path.exists(yaml_filename):
print(f'Skipping {extrafile} as it no longer exists')
continue
with open(yaml_filename, 'r') as extra_yaml:
for pname, project in yaml.safe_load(extra_yaml).items():
for r in project:
if r['repo'].startswith(org + '/'):
repos.add(r['repo'])
print(f'Loaded {len(repos)} repositories from governance.')
return repos
def load_github_repositories(org):
print('Loading GitHub repository list from GitHub...')
gh_repos = {}
for repo in org.get_repos():
gh_repos[repo.full_name] = repo
print(f'Loaded {len(gh_repos)} repositories from GitHub. ')
return gh_repos
def transfer_repository(repofullname, newowner=ARCHIVE_ORG):
# Transfer repository is not yet supported by PyGitHub, so call directly
token = os.environ['GITHUB_TOKEN']
url = f'https://api.github.com/repos/{repofullname}/transfer'
res = requests.post(url,
headers={'Authorization': f'token {token}'},
json={'new_owner': newowner})
if res.status_code != 202:
raise github.GithubException(res.status_code, res.text)
def archive_repository(archive_org, shortname):
repository = archive_org.get_repo(shortname)
repository.edit(archived=True)
def create_repository(org, repofullname, desc, homepage):
name = reponame(org, repofullname)
org.create_repo(name=name,
description=desc,
homepage=homepage,
has_issues=False,
has_projects=False,
has_wiki=False,
team_id=DEFAULT_TEAM_ID)
def update_repository(org, repofullname, desc, homepage):
name = reponame(org, repofullname)
repository = org.get_repo(name)
repository.edit(description=desc, homepage=homepage)
def close_pull_request(org, repofullname, github_pr):
github_pr.create_issue_comment(
PR_CLOSING_TEXT % {'url': homepage(repofullname)})
github_pr.edit(state="closed")
def main(args=sys.argv[1:]):
parser = argparse.ArgumentParser()
parser.add_argument(
'project_config',
help='directory containing the project-config repository')
parser.add_argument(
'governance',
help='directory containing the governance repository')
parser.add_argument(
'--dryrun',
default=False,
help='do not actually do anything',
action='store_true')
args = parser.parse_args(args)
try:
gh = github.Github(os.environ['GITHUB_TOKEN'])
org = gh.get_organization('openstack')
archive_org = gh.get_organization(ARCHIVE_ORG)
except KeyError:
print('Aborting: missing GITHUB_TOKEN environment variable')
sys.exit(1)
if args.dryrun:
print('Running in dry run mode, no action will be actually taken')
# Load data from Gerrit and GitHub
gerrit_descriptions = load_from_project_config(args.project_config)
in_governance = list_repos_in_governance(args.governance)
in_zuul = list_repos_in_zuul(args.project_config)
if in_governance != in_zuul:
print("\nWarning, projects defined in Zuul do not match governance:")
print("\nIn Governance but not in Zuul (should be cleaned up):")
for repo in in_governance.difference(in_zuul):
print(repo)
print("\nIn Zuul but not in Governance (retirement in progress?):")
for repo in in_zuul.difference(in_governance):
print(repo)
print()
github_repos = load_github_repositories(org)
in_github = set(github_repos.keys())
print("\nUpdating repository descriptions:")
for repo in in_github.intersection(in_zuul):
if ((github_repos[repo].description != gerrit_descriptions[repo])
or (github_repos[repo].homepage != homepage(repo))):
print(repo, end=' - ', flush=True)
if args.dryrun:
print('nothing done (--dryrun)')
else:
update_repository(org, repo,
gerrit_descriptions[repo],
homepage(repo))
print('description updated')
time.sleep(1)
print("\nArchiving repositories that are in GitHub but not in Zuul:")
for repo in in_github.difference(in_zuul):
print(repo, end=' - ', flush=True)
if args.dryrun:
print('nothing done (--dryrun)')
else:
transfer_repository(repo)
print(f'moved to {ARCHIVE_ORG}', end=' - ', flush=True)
time.sleep(10)
archive_repository(archive_org, reponame(org, repo))
print('archived')
time.sleep(1)
print("\nCreating repos that are in Zuul & governance but not in Github:")
for repo in (in_zuul.intersection(in_governance)).difference(in_github):
print(repo, end=' - ', flush=True)
if args.dryrun:
print('nothing done (--dryrun)')
else:
create_repository(org, repo,
gerrit_descriptions[repo],
homepage(repo))
print('created')
time.sleep(1)
print("\nIterating through all Github repositories to close PRs:")
for repo, gh_repository in github_repos.items():
for req in gh_repository.get_pulls("open"):
print(f'Closing PR#{req.number} in {repo}', end=' - ', flush=True)
if args.dryrun:
print('nothing done (--dryrun)')
else:
close_pull_request(org, repo, req)
print('closed')
time.sleep(1)
print("Done.")
if __name__ == '__main__':
main()