0f12fa2d5a
Following the deletion of the foundation-board-repos.yaml file, the script failed as it was expecting it. Remove that file from the scanned list, and make the script more resilient in case other extra files go missing in the future (like user-committee.yaml which should soon be cleaned up). Change-Id: I83cff14f19a829b5e56771442ffb3a0341e82d69
276 lines
10 KiB
Python
Executable File
276 lines
10 KiB
Python
Executable File
#!/usr/bin/python3
|
|
#
|
|
# Copyright 2020 Thierry Carrez <thierry@openstack.org>
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
import github
|
|
import requests
|
|
from requests.packages import urllib3
|
|
import yaml
|
|
|
|
|
|
# Turn of warnings about bad SSL config.
|
|
# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
|
|
urllib3.disable_warnings()
|
|
|
|
DESC_SUFFIX = ' Mirror of code maintained at opendev.org.'
|
|
ARCHIVE_ORG = 'openstack-archive'
|
|
DEFAULT_TEAM_ID = 73197
|
|
PR_CLOSING_TEXT = (
|
|
"Thank you for your contribution!\n\n"
|
|
"This GitHub repository is just a mirror of %(url)s, where development "
|
|
"really happens. Pull requests proposed on GitHub are automatically "
|
|
"closed.\n\n"
|
|
"If you are interested in pushing this code upstream, please note "
|
|
"that OpenStack development uses Gerrit for change proposal and code "
|
|
"review.\n\n"
|
|
"If you have never contributed to OpenStack before, please see:\n"
|
|
"https://docs.opendev.org/opendev/infra-manual/latest/gettingstarted.html"
|
|
"\n\nFeel free to reach out on the #openstack-upstream-institute channel "
|
|
"on Freenode IRC in case you need help. You can access it through:\n"
|
|
"https://webchat.freenode.net/#openstack-upstream-institute\n"
|
|
)
|
|
|
|
|
|
def homepage(repo):
|
|
return 'https://opendev.org/' + repo
|
|
|
|
|
|
def reponame(org, fullname):
|
|
owner, name = fullname.split('/')
|
|
if org.login != owner:
|
|
raise ValueError(f'{fullname} does not match target org ({org.login})')
|
|
return name
|
|
|
|
|
|
def load_from_project_config(project_config, org='openstack'):
|
|
print('Loading project config...')
|
|
pc = {}
|
|
proj_yaml_filename = os.path.join(project_config, 'gerrit/projects.yaml')
|
|
with open(proj_yaml_filename, 'r') as proj_yaml:
|
|
for project in yaml.safe_load(proj_yaml):
|
|
if project['project'].startswith(org + '/'):
|
|
desc = project.get('description', '')
|
|
if desc and desc[-1:] != '.':
|
|
desc += '.'
|
|
desc += DESC_SUFFIX
|
|
pc[project['project']] = desc
|
|
print(f'\rLoaded {len(pc)} project descriptions from configuration.')
|
|
return pc
|
|
|
|
|
|
def list_repos_in_zuul(project_config, tenant='openstack', org='openstack'):
|
|
print('Loading Zuul repos...')
|
|
repos = set()
|
|
main_yaml_filename = os.path.join(project_config, 'zuul/main.yaml')
|
|
with open(main_yaml_filename, 'r') as main_yaml:
|
|
for t in yaml.safe_load(main_yaml):
|
|
if t['tenant']['name'] == tenant:
|
|
for ps, pt in t['tenant']['source']['gerrit'].items():
|
|
for elem in pt:
|
|
if type(elem) is dict:
|
|
for k, v in elem.items():
|
|
if k.startswith(org + '/'):
|
|
repos.add(k)
|
|
else:
|
|
if elem.startswith(org + '/'):
|
|
repos.add(elem)
|
|
print(f'Loaded {len(repos)} repositories from Gerrit.')
|
|
return repos
|
|
|
|
|
|
def list_repos_in_governance(governance, org='openstack'):
|
|
print('Loading governance repos...')
|
|
repos = set()
|
|
|
|
proj_yaml_filename = os.path.join(governance, 'reference/projects.yaml')
|
|
with open(proj_yaml_filename, 'r') as proj_yaml:
|
|
for pname, project in yaml.safe_load(proj_yaml).items():
|
|
for dname, deliv in project.get('deliverables', dict()).items():
|
|
for r in deliv['repos']:
|
|
if r.startswith(org + '/'):
|
|
repos.add(r)
|
|
|
|
extrafiles = ['sigs-repos.yaml',
|
|
'technical-committee-repos.yaml',
|
|
'user-committee-repos.yaml']
|
|
for extrafile in extrafiles:
|
|
yaml_filename = os.path.join(governance, 'reference', extrafile)
|
|
if not os.path.exists(yaml_filename):
|
|
print(f'Skipping {extrafile} as it no longer exists')
|
|
continue
|
|
with open(yaml_filename, 'r') as extra_yaml:
|
|
for pname, project in yaml.safe_load(extra_yaml).items():
|
|
for r in project:
|
|
if r['repo'].startswith(org + '/'):
|
|
repos.add(r['repo'])
|
|
print(f'Loaded {len(repos)} repositories from governance.')
|
|
return repos
|
|
|
|
|
|
def load_github_repositories(org):
|
|
print('Loading GitHub repository list from GitHub...')
|
|
gh_repos = {}
|
|
for repo in org.get_repos():
|
|
gh_repos[repo.full_name] = repo
|
|
print(f'Loaded {len(gh_repos)} repositories from GitHub. ')
|
|
return gh_repos
|
|
|
|
|
|
def transfer_repository(repofullname, newowner=ARCHIVE_ORG):
|
|
# Transfer repository is not yet supported by PyGitHub, so call directly
|
|
token = os.environ['GITHUB_TOKEN']
|
|
url = f'https://api.github.com/repos/{repofullname}/transfer'
|
|
res = requests.post(url,
|
|
headers={'Authorization': f'token {token}'},
|
|
json={'new_owner': newowner})
|
|
if res.status_code != 202:
|
|
raise github.GithubException(res.status_code, res.text)
|
|
|
|
|
|
def archive_repository(archive_org, shortname):
|
|
repository = archive_org.get_repo(shortname)
|
|
repository.edit(archived=True)
|
|
|
|
|
|
def create_repository(org, repofullname, desc, homepage):
|
|
name = reponame(org, repofullname)
|
|
org.create_repo(name=name,
|
|
description=desc,
|
|
homepage=homepage,
|
|
has_issues=False,
|
|
has_projects=False,
|
|
has_wiki=False,
|
|
team_id=DEFAULT_TEAM_ID)
|
|
|
|
|
|
def update_repository(org, repofullname, desc, homepage):
|
|
name = reponame(org, repofullname)
|
|
repository = org.get_repo(name)
|
|
repository.edit(description=desc, homepage=homepage)
|
|
|
|
|
|
def close_pull_request(org, repofullname, github_pr):
|
|
github_pr.create_issue_comment(
|
|
PR_CLOSING_TEXT % {'url': homepage(repofullname)})
|
|
github_pr.edit(state="closed")
|
|
|
|
|
|
def main(args=sys.argv[1:]):
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'project_config',
|
|
help='directory containing the project-config repository')
|
|
parser.add_argument(
|
|
'governance',
|
|
help='directory containing the governance repository')
|
|
parser.add_argument(
|
|
'--dryrun',
|
|
default=False,
|
|
help='do not actually do anything',
|
|
action='store_true')
|
|
args = parser.parse_args(args)
|
|
|
|
try:
|
|
gh = github.Github(os.environ['GITHUB_TOKEN'])
|
|
org = gh.get_organization('openstack')
|
|
archive_org = gh.get_organization(ARCHIVE_ORG)
|
|
except KeyError:
|
|
print('Aborting: missing GITHUB_TOKEN environment variable')
|
|
sys.exit(1)
|
|
|
|
if args.dryrun:
|
|
print('Running in dry run mode, no action will be actually taken')
|
|
|
|
# Load data from Gerrit and GitHub
|
|
gerrit_descriptions = load_from_project_config(args.project_config)
|
|
in_governance = list_repos_in_governance(args.governance)
|
|
in_zuul = list_repos_in_zuul(args.project_config)
|
|
|
|
if in_governance != in_zuul:
|
|
print("\nWarning, projects defined in Zuul do not match governance:")
|
|
|
|
print("\nIn Governance but not in Zuul (should be cleaned up):")
|
|
for repo in in_governance.difference(in_zuul):
|
|
print(repo)
|
|
|
|
print("\nIn Zuul but not in Governance (retirement in progress?):")
|
|
for repo in in_zuul.difference(in_governance):
|
|
print(repo)
|
|
print()
|
|
|
|
github_repos = load_github_repositories(org)
|
|
in_github = set(github_repos.keys())
|
|
|
|
print("\nUpdating repository descriptions:")
|
|
for repo in in_github.intersection(in_zuul):
|
|
if ((github_repos[repo].description != gerrit_descriptions[repo])
|
|
or (github_repos[repo].homepage != homepage(repo))):
|
|
print(repo, end=' - ', flush=True)
|
|
if args.dryrun:
|
|
print('nothing done (--dryrun)')
|
|
else:
|
|
update_repository(org, repo,
|
|
gerrit_descriptions[repo],
|
|
homepage(repo))
|
|
print('description updated')
|
|
time.sleep(1)
|
|
|
|
print("\nArchiving repositories that are in GitHub but not in Zuul:")
|
|
for repo in in_github.difference(in_zuul):
|
|
print(repo, end=' - ', flush=True)
|
|
if args.dryrun:
|
|
print('nothing done (--dryrun)')
|
|
else:
|
|
transfer_repository(repo)
|
|
print(f'moved to {ARCHIVE_ORG}', end=' - ', flush=True)
|
|
time.sleep(10)
|
|
archive_repository(archive_org, reponame(org, repo))
|
|
print('archived')
|
|
time.sleep(1)
|
|
|
|
print("\nCreating repos that are in Zuul & governance but not in Github:")
|
|
for repo in (in_zuul.intersection(in_governance)).difference(in_github):
|
|
print(repo, end=' - ', flush=True)
|
|
if args.dryrun:
|
|
print('nothing done (--dryrun)')
|
|
else:
|
|
create_repository(org, repo,
|
|
gerrit_descriptions[repo],
|
|
homepage(repo))
|
|
print('created')
|
|
time.sleep(1)
|
|
|
|
print("\nIterating through all Github repositories to close PRs:")
|
|
for repo, gh_repository in github_repos.items():
|
|
for req in gh_repository.get_pulls("open"):
|
|
print(f'Closing PR#{req.number} in {repo}', end=' - ', flush=True)
|
|
if args.dryrun:
|
|
print('nothing done (--dryrun)')
|
|
else:
|
|
close_pull_request(org, repo, req)
|
|
print('closed')
|
|
time.sleep(1)
|
|
print("Done.")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|