220 lines
8.4 KiB
Python
220 lines
8.4 KiB
Python
#!/usr/bin/python3
|
|
|
|
# Copyright (c) 2019 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
import yaml
|
|
|
|
|
|
def run(commandlist):
|
|
"""Wrapper to run a shell command and return a list of stdout lines."""
|
|
(o, x) = subprocess.Popen(
|
|
commandlist, env=gitenv, stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL).communicate()
|
|
return o.decode('utf-8').strip().split('\n')
|
|
|
|
|
|
class EncryptedPKCS1_OAEP(yaml.YAMLObject):
|
|
"""Causes pyyaml to skip custom YAML tags Zuul groks."""
|
|
yaml_tag = u'!encrypted/pkcs1-oaep'
|
|
yaml_loader = yaml.SafeLoader
|
|
|
|
def __init__(self, x):
|
|
pass
|
|
|
|
@classmethod
|
|
def from_yaml(cls, loader, node):
|
|
return cls(node.value)
|
|
|
|
|
|
# the gerrit git directory
|
|
top = sys.argv[1]
|
|
|
|
# the repo renames file and a corresponding regex for finding them
|
|
renames = {}
|
|
for repo in yaml.safe_load(open(sys.argv[2]))['repos']:
|
|
renames[repo['old']] = repo['new']
|
|
renames_regex = re.compile(
|
|
'([^a-z0-9_-]|^)(%s)([^a-z0-9_-]|$)' % '|'.join(renames.keys()))
|
|
|
|
# our custom git author/committer used by the run function
|
|
gitenv = dict(os.environ)
|
|
gitenv.update({
|
|
'GIT_AUTHOR_NAME': 'OpenDev Sysadmins',
|
|
'GIT_AUTHOR_EMAIL': 'openstack-infra@lists.openstack.org',
|
|
'GIT_COMMITTER_NAME': 'OpenDev Sysadmins',
|
|
'GIT_COMMITTER_EMAIL': 'openstack-infra@lists.openstack.org',
|
|
})
|
|
|
|
# commit message string for generated commits
|
|
commit_message = """\
|
|
OpenDev Migration Patch
|
|
|
|
This commit was bulk generated and pushed by the OpenDev sysadmins
|
|
as a part of the Git hosting and code review systems migration
|
|
detailed in these mailing list posts:
|
|
|
|
http://lists.openstack.org/pipermail/openstack-discuss/2019-March/003603.html
|
|
http://lists.openstack.org/pipermail/openstack-discuss/2019-April/004920.html
|
|
|
|
Attempts have been made to correct repository namespaces and
|
|
hostnames based on simple pattern matching, but it's possible some
|
|
were updated incorrectly or missed entirely. Please reach out to us
|
|
via the contact information listed at https://opendev.org/ with any
|
|
questions you may have.
|
|
"""
|
|
|
|
# find all second-level directories on which we will operate
|
|
repos = run(['find', top, '-maxdepth', '2', '-mindepth', '2', '-name', '*.git', '-type', 'd'])
|
|
|
|
# iterate over each repo
|
|
for bare in repos:
|
|
# clone the repo into a temporary working tree
|
|
with tempfile.TemporaryDirectory() as repodir:
|
|
run(['git', 'clone', bare, repodir])
|
|
origdir = os.getcwd()
|
|
os.chdir(repodir)
|
|
|
|
# build a list of branches for this repo
|
|
branches = []
|
|
branchdump = run(['git', 'branch', '-a'])
|
|
|
|
# iterate over each branch
|
|
for line in branchdump:
|
|
branch = re.match('^remotes/origin/([^ ]+)$', line.strip())
|
|
if branch:
|
|
branches.append(branch.group(1))
|
|
for branch in branches:
|
|
run(['git', 'checkout', '-B', branch, 'origin/' + branch])
|
|
|
|
# build up a list of files to edit
|
|
editfiles = set()
|
|
|
|
# find zuul configs and add ansible playbooks they reference
|
|
zuulfiles = run([
|
|
'find', '.zuul.d/', 'zuul.d/', '.zuul.yaml', 'zuul.yaml',
|
|
'-name', '*.yaml', '-type', 'f'])
|
|
for zuulfile in zuulfiles:
|
|
if zuulfile:
|
|
conf = yaml.safe_load(open(zuulfile))
|
|
if not conf:
|
|
# some repos have empty zuul configs
|
|
continue
|
|
for node in conf:
|
|
if 'job' in node:
|
|
for subnode in ('post-run', 'pre-run', 'run'):
|
|
if subnode in node['job']:
|
|
if type(node['job'][subnode]) is list:
|
|
editfiles.update(node['job'][subnode])
|
|
else:
|
|
editfiles.add(node['job'][subnode])
|
|
|
|
# if there are roles dirs relative to the playbooks, add them too
|
|
for playbook in list(editfiles):
|
|
rolesdir = os.path.join(os.path.dirname(playbook), 'roles')
|
|
if os.path.isdir(rolesdir):
|
|
editfiles.update(run([
|
|
'find', rolesdir, '-type', 'f', '(', '-name', '*.j2',
|
|
'-o', '-name', '*.yaml', '-o', '-name', '*.yml', ')']))
|
|
|
|
# zuul looks at the top level roles dir too
|
|
editfiles.update(run([
|
|
'find', 'roles', '-type', 'f', '(', '-name', '*.j2', '-o',
|
|
'-name', '*.yaml', '-o', '-name', '*.yml', ')']))
|
|
|
|
# and add the zuul configs themselves
|
|
editfiles.update(zuulfiles)
|
|
|
|
# and add .gitreview of course
|
|
editfiles.add('.gitreview')
|
|
|
|
# and zuul/main.yaml so we catch the tenant config
|
|
editfiles.add('zuul/main.yaml')
|
|
|
|
# and gerrit/projects.yaml for manage-projects
|
|
editfiles.add('gerrit/projects.yaml')
|
|
|
|
# and gerritbot/channels.yaml for gerritbot
|
|
editfiles.add('gerritbot/channels.yaml')
|
|
|
|
# drop any empty filename we ended up with
|
|
editfiles.discard('')
|
|
|
|
# read through each file and replace specific patterns
|
|
for fname in editfiles:
|
|
if not os.path.exists(fname):
|
|
continue
|
|
with open(fname) as rfd, tempfile.NamedTemporaryFile() as wfd:
|
|
# track modifications for efficiency
|
|
modified = False
|
|
for line in rfd:
|
|
# apply renames from the mapping
|
|
found = renames_regex.search(line)
|
|
while found:
|
|
line = line.replace(
|
|
found.group(2), renames[found.group(2)])
|
|
modified = True
|
|
found = renames_regex.search(line)
|
|
|
|
# same for git.openstack.org -> opendev.org
|
|
found = re.search("git\.openstack\.org", line)
|
|
while found:
|
|
line = line.replace(
|
|
"git.openstack.org", "opendev.org")
|
|
modified = True
|
|
found = renames_regex.search(line)
|
|
|
|
# and review.openstack.org -> review.opendev.org
|
|
found = re.search("review\.openstack\.org", line)
|
|
while found:
|
|
line = line.replace(
|
|
"review.openstack.org", "review.opendev.org")
|
|
modified = True
|
|
found = renames_regex.search(line)
|
|
|
|
wfd.write(line.encode('utf-8'))
|
|
|
|
# copy any modified file back into the worktree
|
|
if modified:
|
|
wfd.flush()
|
|
shutil.copyfile(wfd.name, fname)
|
|
modified = False
|
|
|
|
# special logic to rename Gerrit ACL files
|
|
if bare.endswith('/project-config.git'):
|
|
for acl in run(['git', 'ls-files', 'gerrit/acls/']):
|
|
found = renames_regex.search(acl)
|
|
if found:
|
|
newpath = acl.replace(
|
|
found.group(2), renames[found.group(2)])
|
|
os.makedirs(os.path.dirname(newpath), exist_ok=True)
|
|
run(['git', 'mv', acl, newpath])
|
|
|
|
# commit and push our changes, if there are any
|
|
if run(['git', 'diff']):
|
|
with tempfile.NamedTemporaryFile() as message:
|
|
message.write(commit_message.encode('utf-8'))
|
|
message.flush()
|
|
run(['git', 'commit', '-a', '-F', message.name])
|
|
run(['git', 'push', 'origin', 'HEAD'])
|
|
|
|
# switch back before the context manager deletes our cwd
|
|
os.chdir(origdir)
|