system-config/tools/opendev-migrate

#!/usr/bin/python3

# Copyright (c) 2019 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.

import csv
import io
import json
import requests
import yaml

# this will hold our mapping of old names(paces) to new
moves = {}

# here's a list of all (non-meta)projects in gerrit
repos = [r for r in json.loads(requests.get(
    'http://review.openstack.org/projects/').text[5:]).keys() if '/' in r]

# a map of the first pair of columns from the namespace request ethercalc
overrides = dict([r[:2] for r in csv.reader(io.StringIO(
    requests.get('https://ethercalc.openstack.org/opendev-transition.csv').text
    )) if '/' in r[1]])

# all projects which are officially governed by openstack or osf
openstack = []
o_gov = 'https://opendev.org/openstack/governance/raw/branch/master/reference/'
data = yaml.safe_load(requests.get(o_gov + 'projects.yaml').text)
for team in data.values():
    for deli in team['deliverables'].values():
        for repo in deli['repos']:
            openstack.append(repo)
for f in ('foundation-board-repos.yaml', 'sigs-repos.yaml',
          'technical-committee-repos.yaml', 'user-committee-repos.yaml'):
    data = yaml.safe_load(requests.get(o_gov + f).text)
    for team in data.values():
        for repo in team:
            openstack.append(repo['repo'])

# projects which were at one time officially governed by openstack
openstack_legacy = []
data = yaml.safe_load(requests.get(o_gov + 'legacy.yaml').text)
for team in data.values():
    for deli in team['deliverables'].values():
        for repo in deli['repos']:
            openstack_legacy.append(repo)

# use the jeepyb config to identify whitelabeled oip git projects
airship = []
starlingx = []
zuul = []
data = yaml.safe_load(requests.get(
    'https://opendev.org/openstack-infra/project-config/raw/branch/master/'
    'gerrit/projects.yaml').text)
for project in data:
    if 'cgit-alias' in project:
        if project['cgit-alias']['site'] == 'git.airshipit.org':
            airship.append(project['project'])
        elif project['cgit-alias']['site'] == 'git.starlingx.io':
            starlingx.append(project['project'])
        elif project['cgit-alias']['site'] == 'git.zuul-ci.org':
            zuul.append(project['project'])

for repo in repos:
    # apply the requested namespace overrides first
    if repo in overrides:
        moves[repo] = overrides[repo]

    # airship repos identified drop the airship- prefix and move to airship
    elif repo in airship:
        moves[repo] = 'airship/' + repo.split('/')[1].replace('airship-', '')

    # starlingx repos drop the stx- prefix and move to starlingx
    elif repo in starlingx:
        moves[repo] = 'starlingx/' + repo.split('/')[1].replace('stx-', '')

    # all current openstack repos move to openstack
    elif repo in openstack:
        moves[repo] = 'openstack/' + repo.split('/')[1]

    # zuul repos move to zuul
    elif repo in zuul:
        moves[repo] = 'zuul/' + repo.split('/')[1]

    # former openstack repositories which aren't accounted for go in openstack
    elif repo in openstack_legacy:
        moves[repo] = 'openstack/' + repo.split('/')[1]

    # unofficial repositories move from openstack to x
    elif repo.startswith('openstack/'):
        moves[repo] = 'x/' + repo.split('/')[1]

    # everything else is unchanged
    else:
        moves[repo] = repo

# we'll use this data structure for the rename_repos playbook input
output = {'repos': []}

for mapping in moves.items():
    if mapping[0] != mapping[1]:
        # convenient stdout feedback is for sharing with people
        print('%s -> %s' % mapping)

        # update the rename_repos data structure
        output['repos'].append({'old': mapping[0], 'new': mapping[1]})

# https://docs.openstack.org/infra/system-config/gerrit.html#renaming-a-project
with open('repos.yaml', 'w') as outfile:
    yaml.dump(output, outfile)

# We should add this to the rename playbook, but time is short
with open('zuul-rename.sh', 'w') as outfile:
    keyroot = '/var/lib/zuul/keys'
    for d in output['repos']:
        outfile.write('mv %s/ssh/project/gerrit/%s %s/ssh/project/gerrit/%s\n' %
                      (keyroot, d['old'], keyroot, d['new']))
        outfile.write('mv %s/secrets/project/gerrit/%s %s/secrets/project/gerrit/%s\n' %
                      (keyroot, d['old'], keyroot, d['new']))