Define maintain-github-openstack-mirror job

Opendev no longer automatically creates repositories on the
GitHub mirror, nor does it update descriptions or closes open PRs.

Add a playbook and a job for periodically maintaining the GitHub
mirror for the 'openstack' organization:

- updating descriptions based on Gerrit project descriptions
- creating on GitHub newly-added openstack repositories
- archiving from GitHub recently-retired openstack repositories
- closing any open PR with a healpful message

This job makes use of a GitHub API token (from the openstack-mirroring
user) and is defined to run periodically on project-config.

Change-Id: Ic02f436eb655dcbe84824b304ea2933e16312e67

playbooks/maintain-github-mirror/github_manager.py

@@ -0,0 +1,273 @@
+#!/usr/bin/python3
+#
+# Copyright 2020 Thierry Carrez <thierry@openstack.org>
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import argparse
+import os
+import sys
+import time
+
+import github
+import requests
+from requests.packages import urllib3
+import yaml
+
+
+# Turn of warnings about bad SSL config.
+# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
+urllib3.disable_warnings()
+
+DESC_SUFFIX = ' Mirror of code maintained at opendev.org.'
+ARCHIVE_ORG = 'openstack-archive'
+DEFAULT_TEAM_ID = 73197
+PR_CLOSING_TEXT = (
+    "Thank you for your contribution!\n\n"
+    "This GitHub repository is just a mirror of %(url)s, where development "
+    "really happens. Pull requests proposed on GitHub are automatically "
+    "closed.\n\n"
+    "If you are interested in pushing this code upstream, please note "
+    "that OpenStack development uses Gerrit for change proposal and code "
+    "review.\n\n"
+    "If you have never contributed to OpenStack before, please see:\n"
+    "https://docs.opendev.org/opendev/infra-manual/latest/gettingstarted.html"
+    "\n\nFeel free to reach out on the #openstack-upstream-institute channel "
+    "on Freenode IRC in case you need help. You can access it through:\n"
+    "https://webchat.freenode.net/#openstack-upstream-institute\n"
+)
+
+
+def homepage(repo):
+    return 'https://opendev.org/' + repo
+
+
+def reponame(org, fullname):
+    owner, name = fullname.split('/')
+    if org.login != owner:
+        raise ValueError(f'{fullname} does not match target org ({org.login})')
+    return name
+
+
+def load_from_project_config(project_config, org='openstack'):
+    print('Loading project config...')
+    pc = {}
+    proj_yaml_filename = os.path.join(project_config, 'gerrit/projects.yaml')
+    with open(proj_yaml_filename, 'r') as proj_yaml:
+        for project in yaml.safe_load(proj_yaml):
+            if project['project'].startswith(org + '/'):
+                desc = project.get('description', '')
+                if desc and desc[-1:] != '.':
+                    desc += '.'
+                desc += DESC_SUFFIX
+                pc[project['project']] = desc
+    print(f'\rLoaded {len(pc)} project descriptions from configuration.')
+    return pc
+
+
+def list_repos_in_zuul(project_config, tenant='openstack', org='openstack'):
+    print('Loading Zuul repos...')
+    repos = set()
+    main_yaml_filename = os.path.join(project_config, 'zuul/main.yaml')
+    with open(main_yaml_filename, 'r') as main_yaml:
+        for t in yaml.safe_load(main_yaml):
+            if t['tenant']['name'] == tenant:
+                for ps, pt in t['tenant']['source']['gerrit'].items():
+                    for elem in pt:
+                        if type(elem) is dict:
+                            for k, v in elem.items():
+                                if k.startswith(org + '/'):
+                                    repos.add(k)
+                        else:
+                            if elem.startswith(org + '/'):
+                                repos.add(elem)
+    print(f'Loaded {len(repos)} repositories from Gerrit.')
+    return repos
+
+
+def list_repos_in_governance(governance, org='openstack'):
+    print('Loading governance repos...')
+    repos = set()
+
+    proj_yaml_filename = os.path.join(governance, 'reference/projects.yaml')
+    with open(proj_yaml_filename, 'r') as proj_yaml:
+        for pname, project in yaml.safe_load(proj_yaml).items():
+            for dname, deliv in project.get('deliverables', dict()).items():
+                for r in deliv['repos']:
+                    if r.startswith(org + '/'):
+                        repos.add(r)
+
+    extrafiles = ['sigs-repos.yaml',
+                  'foundation-board-repos.yaml',
+                  'technical-committee-repos.yaml',
+                  'user-committee-repos.yaml']
+    for extrafile in extrafiles:
+        yaml_filename = os.path.join(governance, 'reference', extrafile)
+        with open(yaml_filename, 'r') as extra_yaml:
+            for pname, project in yaml.safe_load(extra_yaml).items():
+                for r in project:
+                    if r['repo'].startswith(org + '/'):
+                        repos.add(r['repo'])
+    print(f'Loaded {len(repos)} repositories from governance.')
+    return repos
+
+
+def load_github_repositories(org):
+    print('Loading GitHub repository list from GitHub...')
+    gh_repos = {}
+    for repo in org.get_repos():
+        gh_repos[repo.full_name] = repo
+    print(f'Loaded {len(gh_repos)} repositories from GitHub.     ')
+    return gh_repos
+
+
+def transfer_repository(repofullname, newowner=ARCHIVE_ORG):
+    # Transfer repository is not yet supported by PyGitHub, so call directly
+    token = os.environ['GITHUB_TOKEN']
+    url = f'https://api.github.com/repos/{repofullname}/transfer'
+    res = requests.post(url,
+                        headers={'Authorization': f'token {token}'},
+                        json={'new_owner': newowner})
+    if res.status_code != 202:
+        raise github.GithubException(res.status_code, res.text)
+
+
+def archive_repository(archive_org, shortname):
+    repository = archive_org.get_repo(shortname)
+    repository.edit(archived=True)
+
+
+def create_repository(org, repofullname, desc, homepage):
+    name = reponame(org, repofullname)
+    org.create_repo(name=name,
+                    description=desc,
+                    homepage=homepage,
+                    has_issues=False,
+                    has_projects=False,
+                    has_wiki=False,
+                    team_id=DEFAULT_TEAM_ID)
+
+
+def update_repository(org, repofullname, desc, homepage):
+    name = reponame(org, repofullname)
+    repository = org.get_repo(name)
+    repository.edit(description=desc, homepage=homepage)
+
+
+def close_pull_request(org, repofullname, github_pr):
+    github_pr.create_issue_comment(
+        PR_CLOSING_TEXT % {'url': homepage(repofullname)})
+    github_pr.edit(state="closed")
+
+
+def main(args=sys.argv[1:]):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'project_config',
+        help='directory containing the project-config repository')
+    parser.add_argument(
+        'governance',
+        help='directory containing the governance repository')
+    parser.add_argument(
+        '--dryrun',
+        default=False,
+        help='do not actually do anything',
+        action='store_true')
+    args = parser.parse_args(args)
+
+    try:
+        gh = github.Github(os.environ['GITHUB_TOKEN'])
+        org = gh.get_organization('openstack')
+        archive_org = gh.get_organization(ARCHIVE_ORG)
+    except KeyError:
+        print('Aborting: missing GITHUB_TOKEN environment variable')
+        sys.exit(1)
+
+    if args.dryrun:
+        print('Running in dry run mode, no action will be actually taken')
+
+    # Load data from Gerrit and GitHub
+    gerrit_descriptions = load_from_project_config(args.project_config)
+    in_governance = list_repos_in_governance(args.governance)
+    in_zuul = list_repos_in_zuul(args.project_config)
+
+    if in_governance != in_zuul:
+        print("\nWarning, projects defined in Zuul do not match governance:")
+
+        print("\nIn Governance but not in Zuul (should be cleaned up):")
+        for repo in in_governance.difference(in_zuul):
+            print(repo)
+
+        print("\nIn Zuul but not in Governance (retirement in progress?):")
+        for repo in in_zuul.difference(in_governance):
+            print(repo)
+        print()
+
+    github_repos = load_github_repositories(org)
+    in_github = set(github_repos.keys())
+
+    print("\nUpdating repository descriptions:")
+    for repo in in_github.intersection(in_zuul):
+        if ((github_repos[repo].description != gerrit_descriptions[repo])
+           or (github_repos[repo].homepage != homepage(repo))):
+            print(repo, end=' - ', flush=True)
+            if args.dryrun:
+                print('nothing done (--dryrun)')
+            else:
+                update_repository(org, repo,
+                                  gerrit_descriptions[repo],
+                                  homepage(repo))
+                print('description updated')
+                time.sleep(1)
+
+    print("\nArchiving repositories that are in GitHub but not in Zuul:")
+    for repo in in_github.difference(in_zuul):
+        print(repo, end=' - ', flush=True)
+        if args.dryrun:
+            print('nothing done (--dryrun)')
+        else:
+            transfer_repository(repo)
+            print(f'moved to {ARCHIVE_ORG}', end=' - ', flush=True)
+            time.sleep(10)
+            archive_repository(archive_org, reponame(org, repo))
+            print('archived')
+            time.sleep(1)
+
+    print("\nCreating repos that are in Zuul & governance but not in Github:")
+    for repo in (in_zuul.intersection(in_governance)).difference(in_github):
+        print(repo, end=' - ', flush=True)
+        if args.dryrun:
+            print('nothing done (--dryrun)')
+        else:
+            create_repository(org, repo,
+                              gerrit_descriptions[repo],
+                              homepage(repo))
+            print('created')
+            time.sleep(1)
+
+    print("\nIterating through all Github repositories to close PRs:")
+    for repo, gh_repository in github_repos.items():
+        for req in gh_repository.get_pulls("open"):
+            print(f'Closing PR#{req.number} in {repo}', end=' - ', flush=True)
+            if args.dryrun:
+                print('nothing done (--dryrun)')
+            else:
+                close_pull_request(org, repo, req)
+                print('closed')
+                time.sleep(1)
+    print("Done.")
+
+
+if __name__ == '__main__':
+    main()

playbooks/maintain-github-mirror/run.yaml

@@ -0,0 +1,29 @@
+- hosts: all
+  tasks:
+    - name: Ensure pip
+      include_role:
+        name: ensure-pip
+
+    - name: Copy manager script
+      copy:
+        src: github_manager.py
+        dest: "{{ ansible_user_dir }}"
+
+    - name: Install dependencies
+      pip:
+        name: github
+        virtualenv: "{{ ansible_user_dir }}/.venv"
+        virtualenv_command: "{{ ensure_pip_virtualenv_command }}"
+
+    - name: Run manager script
+      command: "{{ ansible_user_dir }}/.venv/bin/python {{ ansible_user_dir }}/github_manager.py {{ conf }} {{ gov }}"
+      environment:
+        GITHUB_TOKEN: "{{ github_credentials.api_token }}"
+
+    - name: Clean up after ourselves
+      file:
+        path: "{{ item }}"
+        state: absent
+      with_items:
+        - github_manager.py
+        - .venv

zuul.d/jobs.yaml

@@ -788,6 +788,24 @@
         secret: openstack-github-mirroring
         pass-to-parent: true
 
+- job:
+    name: maintain-github-openstack-mirror
+    description: |
+      Runs maintenance scripts for the OpenStack GitHub mirror. The script
+      updates the descriptions, creates new repositories, archives retired
+      ones, and closes any open PR. It is meant to be run periodically.
+    final: true
+    protected: true
+    run: playbooks/maintain-github-mirror/run.yaml
+    required-projects:
+      - name: openstack/governance
+    secrets:
+      - name: github_credentials
+        secret: openstack-github-mirroring
+    vars:
+      conf: "{{ zuul.projects['opendev.org/opendev/project_config'].src_dir }}"
+      gov: "{{ zuul.projects['opendev.org/opendev/governance'].src_dir }}"
+
 - job:
     name: propose-updates
     pre-run: playbooks/proposal/pre.yaml

zuul.d/projects.yaml

@@ -4579,6 +4579,7 @@
     periodic:
       jobs:
         - propose-project-config-update
+        - maintain-github-openstack-mirror
     opendev-prod-hourly:
       jobs:
         - publish-irc-meetings

zuul.d/secrets.yaml

@@ -773,3 +773,14 @@
           xRCWUjMcQUQFKqUB4e3VBY9mOxrqACesPuE5H0avVwIwaPGZngPgDBmDyZDM1X28N9SvZ
           h0x++Jcyt5MX2HULKvnmhMeRuskGrr/4ujaJz+X85j/qAySY0T+euU5QoDaAjRNNQeoaO
           X3PXCRgfNXahgWYgq1Xu5l9wLhXaEpWQuOj6BGVv/Fy2yVR89b/1PryYzRCpoI=
+      api_token: !encrypted/pkcs1-oaep
+        - P4WWrBQZeA/KpKrqjBpU8j6Htql+J3kHbauaQqZVoEkcggj214O45TNSpz40ZqOGZNdbj
+          38/tp7ofTkfUNTMYR27/fngZgHXFTLm9SvLb0lvoML5QthtaWY3UCTkBAlQ5P/XffIFff
+          p3zvYzh8SI+8hFrgr5Wod7xvqBTeQOgsR2pMHb2EJRy29Ru0ZD4DqF2YICdjQsAt9QGYn
+          deD7u5IWfgfeh3CdtYRX9s2NPpBLM1Fz57Nd4ssptjIhOLBw/Q+UhK2W2ZB9fpc7AKVvs
+          EfZObOuU8vzEzhEoRSJrBsjaImi9bwZZmucV4BupZVEaVprOWr7AHgEZZ3R1AtBOFt3D/
+          uhKNO6IstRKioxyqm96k+cSOCVbEhpm/ibktgJR8Im6KiRGez2MENEuGDk0Wu//xyF6/O
+          q28o1lSJIop8n1dxFAmjEaxyarl/GqUWQEh2XErhvDVsT5IRtav0yvihb3v/495SP7qNm
+          HETs35aycMm/KTjhryPoQbsAAmVe/i/+PFcyxcMDPZHQmJcWRD+K7lJb3o18kNST410B/
+          FtiR0LGtDxKM1bdi57Qc3f43P4jzY3Px07SSKVFKSkuI1zSLnsZSbmWg/wBHcjllsA73L
+          l0HItpoMi3S3KDsFajJbk2UE6NhCBD7kmsSB69L6yb7VJdKZqMAHS2BSSXIRdA=