Merge "Add opendev migration repo rename scripts"

This commit is contained in:
Zuul 2019-05-30 21:07:37 +00:00 committed by Gerrit Code Review
commit 216059e134
2 changed files with 347 additions and 0 deletions

128
tools/opendev-migrate Normal file
View File

@ -0,0 +1,128 @@
#!/usr/bin/python3
# Copyright (c) 2019 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
import csv
import io
import json
import requests
import yaml
# this will hold our mapping of old names(paces) to new
moves = {}
# here's a list of all (non-meta)projects in gerrit
repos = [r for r in json.loads(requests.get(
'http://review.openstack.org/projects/').text[5:]).keys() if '/' in r]
# a map of the first pair of columns from the namespace request ethercalc
overrides = dict([r[:2] for r in csv.reader(io.StringIO(
requests.get('https://ethercalc.openstack.org/opendev-transition.csv').text
)) if '/' in r[1]])
# all projects which are officially governed by openstack or osf
openstack = []
o_gov = 'https://opendev.org/openstack/governance/raw/branch/master/reference/'
data = yaml.safe_load(requests.get(o_gov + 'projects.yaml').text)
for team in data.values():
for deli in team['deliverables'].values():
for repo in deli['repos']:
openstack.append(repo)
for f in ('foundation-board-repos.yaml', 'sigs-repos.yaml',
'technical-committee-repos.yaml', 'user-committee-repos.yaml'):
data = yaml.safe_load(requests.get(o_gov + f).text)
for team in data.values():
for repo in team:
openstack.append(repo['repo'])
# projects which were at one time officially governed by openstack
openstack_legacy = []
data = yaml.safe_load(requests.get(o_gov + 'legacy.yaml').text)
for team in data.values():
for deli in team['deliverables'].values():
for repo in deli['repos']:
openstack_legacy.append(repo)
# use the jeepyb config to identify whitelabeled oip git projects
airship = []
starlingx = []
zuul = []
data = yaml.safe_load(requests.get(
'https://opendev.org/openstack-infra/project-config/raw/branch/master/'
'gerrit/projects.yaml').text)
for project in data:
if 'cgit-alias' in project:
if project['cgit-alias']['site'] == 'git.airshipit.org':
airship.append(project['project'])
elif project['cgit-alias']['site'] == 'git.starlingx.io':
starlingx.append(project['project'])
elif project['cgit-alias']['site'] == 'git.zuul-ci.org':
zuul.append(project['project'])
for repo in repos:
# apply the requested namespace overrides first
if repo in overrides:
moves[repo] = overrides[repo]
# airship repos identified drop the airship- prefix and move to airship
elif repo in airship:
moves[repo] = 'airship/' + repo.split('/')[1].replace('airship-', '')
# starlingx repos drop the stx- prefix and move to starlingx
elif repo in starlingx:
moves[repo] = 'starlingx/' + repo.split('/')[1].replace('stx-', '')
# all current openstack repos move to openstack
elif repo in openstack:
moves[repo] = 'openstack/' + repo.split('/')[1]
# zuul repos move to zuul
elif repo in zuul:
moves[repo] = 'zuul/' + repo.split('/')[1]
# former openstack repositories which aren't accounted for go in openstack
elif repo in openstack_legacy:
moves[repo] = 'openstack/' + repo.split('/')[1]
# unofficial repositories move from openstack to x
elif repo.startswith('openstack/'):
moves[repo] = 'x/' + repo.split('/')[1]
# everything else is unchanged
else:
moves[repo] = repo
# we'll use this data structure for the rename_repos playbook input
output = {'repos': []}
for mapping in moves.items():
if mapping[0] != mapping[1]:
# convenient stdout feedback is for sharing with people
print('%s -> %s' % mapping)
# update the rename_repos data structure
output['repos'].append({'old': mapping[0], 'new': mapping[1]})
# https://docs.openstack.org/infra/system-config/gerrit.html#renaming-a-project
with open('repos.yaml', 'w') as outfile:
yaml.dump(output, outfile)
# We should add this to the rename playbook, but time is short
with open('zuul-rename.sh', 'w') as outfile:
keyroot = '/var/lib/zuul/keys'
for d in output['repos']:
outfile.write('mv %s/ssh/project/gerrit/%s %s/ssh/project/gerrit/%s\n' %
(keyroot, d['old'], keyroot, d['new']))
outfile.write('mv %s/secrets/project/gerrit/%s %s/secrets/project/gerrit/%s\n' %
(keyroot, d['old'], keyroot, d['new']))

219
tools/opendev-patching Normal file
View File

@ -0,0 +1,219 @@
#!/usr/bin/python3
# Copyright (c) 2019 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
import os
import re
import shutil
import subprocess
import sys
import tempfile
import yaml
def run(commandlist):
"""Wrapper to run a shell command and return a list of stdout lines."""
(o, x) = subprocess.Popen(
commandlist, env=gitenv, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL).communicate()
return o.decode('utf-8').strip().split('\n')
class EncryptedPKCS1_OAEP(yaml.YAMLObject):
"""Causes pyyaml to skip custom YAML tags Zuul groks."""
yaml_tag = u'!encrypted/pkcs1-oaep'
yaml_loader = yaml.SafeLoader
def __init__(self, x):
pass
@classmethod
def from_yaml(cls, loader, node):
return cls(node.value)
# the gerrit git directory
top = sys.argv[1]
# the repo renames file and a corresponding regex for finding them
renames = {}
for repo in yaml.safe_load(open(sys.argv[2]))['repos']:
renames[repo['old']] = repo['new']
renames_regex = re.compile(
'([^a-z0-9_-]|^)(%s)([^a-z0-9_-]|$)' % '|'.join(renames.keys()))
# our custom git author/committer used by the run function
gitenv = dict(os.environ)
gitenv.update({
'GIT_AUTHOR_NAME': 'OpenDev Sysadmins',
'GIT_AUTHOR_EMAIL': 'openstack-infra@lists.openstack.org',
'GIT_COMMITTER_NAME': 'OpenDev Sysadmins',
'GIT_COMMITTER_EMAIL': 'openstack-infra@lists.openstack.org',
})
# commit message string for generated commits
commit_message = """\
OpenDev Migration Patch
This commit was bulk generated and pushed by the OpenDev sysadmins
as a part of the Git hosting and code review systems migration
detailed in these mailing list posts:
http://lists.openstack.org/pipermail/openstack-discuss/2019-March/003603.html
http://lists.openstack.org/pipermail/openstack-discuss/2019-April/004920.html
Attempts have been made to correct repository namespaces and
hostnames based on simple pattern matching, but it's possible some
were updated incorrectly or missed entirely. Please reach out to us
via the contact information listed at https://opendev.org/ with any
questions you may have.
"""
# find all second-level directories on which we will operate
repos = run(['find', top, '-maxdepth', '2', '-mindepth', '2', '-name', '*.git', '-type', 'd'])
# iterate over each repo
for bare in repos:
# clone the repo into a temporary working tree
with tempfile.TemporaryDirectory() as repodir:
run(['git', 'clone', bare, repodir])
origdir = os.getcwd()
os.chdir(repodir)
# build a list of branches for this repo
branches = []
branchdump = run(['git', 'branch', '-a'])
# iterate over each branch
for line in branchdump:
branch = re.match('^remotes/origin/([^ ]+)$', line.strip())
if branch:
branches.append(branch.group(1))
for branch in branches:
run(['git', 'checkout', '-B', branch, 'origin/' + branch])
# build up a list of files to edit
editfiles = set()
# find zuul configs and add ansible playbooks they reference
zuulfiles = run([
'find', '.zuul.d/', 'zuul.d/', '.zuul.yaml', 'zuul.yaml',
'-name', '*.yaml', '-type', 'f'])
for zuulfile in zuulfiles:
if zuulfile:
conf = yaml.safe_load(open(zuulfile))
if not conf:
# some repos have empty zuul configs
continue
for node in conf:
if 'job' in node:
for subnode in ('post-run', 'pre-run', 'run'):
if subnode in node['job']:
if type(node['job'][subnode]) is list:
editfiles.update(node['job'][subnode])
else:
editfiles.add(node['job'][subnode])
# if there are roles dirs relative to the playbooks, add them too
for playbook in list(editfiles):
rolesdir = os.path.join(os.path.dirname(playbook), 'roles')
if os.path.isdir(rolesdir):
editfiles.update(run([
'find', rolesdir, '-type', 'f', '(', '-name', '*.j2',
'-o', '-name', '*.yaml', '-o', '-name', '*.yml', ')']))
# zuul looks at the top level roles dir too
editfiles.update(run([
'find', 'roles', '-type', 'f', '(', '-name', '*.j2', '-o',
'-name', '*.yaml', '-o', '-name', '*.yml', ')']))
# and add the zuul configs themselves
editfiles.update(zuulfiles)
# and add .gitreview of course
editfiles.add('.gitreview')
# and zuul/main.yaml so we catch the tenant config
editfiles.add('zuul/main.yaml')
# and gerrit/projects.yaml for manage-projects
editfiles.add('gerrit/projects.yaml')
# and gerritbot/channels.yaml for gerritbot
editfiles.add('gerritbot/channels.yaml')
# drop any empty filename we ended up with
editfiles.discard('')
# read through each file and replace specific patterns
for fname in editfiles:
if not os.path.exists(fname):
continue
with open(fname) as rfd, tempfile.NamedTemporaryFile() as wfd:
# track modifications for efficiency
modified = False
for line in rfd:
# apply renames from the mapping
found = renames_regex.search(line)
while found:
line = line.replace(
found.group(2), renames[found.group(2)])
modified = True
found = renames_regex.search(line)
# same for git.openstack.org -> opendev.org
found = re.search("git\.openstack\.org", line)
while found:
line = line.replace(
"git.openstack.org", "opendev.org")
modified = True
found = renames_regex.search(line)
# and review.openstack.org -> review.opendev.org
found = re.search("review\.openstack\.org", line)
while found:
line = line.replace(
"review.openstack.org", "review.opendev.org")
modified = True
found = renames_regex.search(line)
wfd.write(line.encode('utf-8'))
# copy any modified file back into the worktree
if modified:
wfd.flush()
shutil.copyfile(wfd.name, fname)
modified = False
# special logic to rename Gerrit ACL files
if bare.endswith('/project-config.git'):
for acl in run(['git', 'ls-files', 'gerrit/acls/']):
found = renames_regex.search(acl)
if found:
newpath = acl.replace(
found.group(2), renames[found.group(2)])
os.makedirs(os.path.dirname(newpath), exist_ok=True)
run(['git', 'mv', acl, newpath])
# commit and push our changes, if there are any
if run(['git', 'diff']):
with tempfile.NamedTemporaryFile() as message:
message.write(commit_message.encode('utf-8'))
message.flush()
run(['git', 'commit', '-a', '-F', message.name])
run(['git', 'push', 'origin', 'HEAD'])
# switch back before the context manager deletes our cwd
os.chdir(origdir)