#!/usr/bin/python3 # Copyright (c) 2019 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. import os import re import shutil import subprocess import sys import tempfile import yaml def run(commandlist): """Wrapper to run a shell command and return a list of stdout lines.""" (o, x) = subprocess.Popen( commandlist, env=gitenv, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).communicate() return o.decode('utf-8').strip().split('\n') class EncryptedPKCS1_OAEP(yaml.YAMLObject): """Causes pyyaml to skip custom YAML tags Zuul groks.""" yaml_tag = u'!encrypted/pkcs1-oaep' yaml_loader = yaml.SafeLoader def __init__(self, x): pass @classmethod def from_yaml(cls, loader, node): return cls(node.value) # the gerrit git directory top = sys.argv[1] # the repo renames file and a corresponding regex for finding them renames = {} for repo in yaml.safe_load(open(sys.argv[2]))['repos']: renames[repo['old']] = repo['new'] renames_regex = re.compile( '([^a-z0-9_-]|^)(%s)([^a-z0-9_-]|$)' % '|'.join(renames.keys())) # our custom git author/committer used by the run function gitenv = dict(os.environ) gitenv.update({ 'GIT_AUTHOR_NAME': 'OpenDev Sysadmins', 'GIT_AUTHOR_EMAIL': 'openstack-infra@lists.openstack.org', 'GIT_COMMITTER_NAME': 'OpenDev Sysadmins', 'GIT_COMMITTER_EMAIL': 'openstack-infra@lists.openstack.org', }) # commit message string for generated commits commit_message = """\ OpenDev Migration Patch This commit was bulk generated and pushed by the OpenDev sysadmins as a part of the Git hosting and code review systems migration detailed in these mailing list posts: http://lists.openstack.org/pipermail/openstack-discuss/2019-March/003603.html http://lists.openstack.org/pipermail/openstack-discuss/2019-April/004920.html Attempts have been made to correct repository namespaces and hostnames based on simple pattern matching, but it's possible some were updated incorrectly or missed entirely. Please reach out to us via the contact information listed at https://opendev.org/ with any questions you may have. """ # find all second-level directories on which we will operate repos = run(['find', top, '-maxdepth', '2', '-mindepth', '2', '-name', '*.git', '-type', 'd']) # iterate over each repo for bare in repos: # clone the repo into a temporary working tree with tempfile.TemporaryDirectory() as repodir: run(['git', 'clone', bare, repodir]) origdir = os.getcwd() os.chdir(repodir) # build a list of branches for this repo branches = [] branchdump = run(['git', 'branch', '-a']) # iterate over each branch for line in branchdump: branch = re.match('^remotes/origin/([^ ]+)$', line.strip()) if branch: branches.append(branch.group(1)) for branch in branches: run(['git', 'checkout', '-B', branch, 'origin/' + branch]) # build up a list of files to edit editfiles = set() # find zuul configs and add ansible playbooks they reference zuulfiles = run([ 'find', '.zuul.d/', 'zuul.d/', '.zuul.yaml', 'zuul.yaml', '-name', '*.yaml', '-type', 'f']) for zuulfile in zuulfiles: if zuulfile: conf = yaml.safe_load(open(zuulfile)) if not conf: # some repos have empty zuul configs continue for node in conf: if 'job' in node: for subnode in ('post-run', 'pre-run', 'run'): if subnode in node['job']: if type(node['job'][subnode]) is list: editfiles.update(node['job'][subnode]) else: editfiles.add(node['job'][subnode]) # if there are roles dirs relative to the playbooks, add them too for playbook in list(editfiles): rolesdir = os.path.join(os.path.dirname(playbook), 'roles') if os.path.isdir(rolesdir): editfiles.update(run([ 'find', rolesdir, '-type', 'f', '(', '-name', '*.j2', '-o', '-name', '*.yaml', '-o', '-name', '*.yml', ')'])) # zuul looks at the top level roles dir too editfiles.update(run([ 'find', 'roles', '-type', 'f', '(', '-name', '*.j2', '-o', '-name', '*.yaml', '-o', '-name', '*.yml', ')'])) # and add the zuul configs themselves editfiles.update(zuulfiles) # and add .gitreview of course editfiles.add('.gitreview') # and zuul/main.yaml so we catch the tenant config editfiles.add('zuul/main.yaml') # and gerrit/projects.yaml for manage-projects editfiles.add('gerrit/projects.yaml') # and gerritbot/channels.yaml for gerritbot editfiles.add('gerritbot/channels.yaml') # drop any empty filename we ended up with editfiles.discard('') # read through each file and replace specific patterns for fname in editfiles: if not os.path.exists(fname): continue with open(fname) as rfd, tempfile.NamedTemporaryFile() as wfd: # track modifications for efficiency modified = False for line in rfd: # apply renames from the mapping found = renames_regex.search(line) while found: line = line.replace( found.group(2), renames[found.group(2)]) modified = True found = renames_regex.search(line) # same for git.openstack.org -> opendev.org found = re.search("git\.openstack\.org", line) while found: line = line.replace( "git.openstack.org", "opendev.org") modified = True found = renames_regex.search(line) # and review.openstack.org -> review.opendev.org found = re.search("review\.openstack\.org", line) while found: line = line.replace( "review.openstack.org", "review.opendev.org") modified = True found = renames_regex.search(line) wfd.write(line.encode('utf-8')) # copy any modified file back into the worktree if modified: wfd.flush() shutil.copyfile(wfd.name, fname) modified = False # special logic to rename Gerrit ACL files if bare.endswith('/project-config.git'): for acl in run(['git', 'ls-files', 'gerrit/acls/']): found = renames_regex.search(acl) if found: newpath = acl.replace( found.group(2), renames[found.group(2)]) os.makedirs(os.path.dirname(newpath), exist_ok=True) run(['git', 'mv', acl, newpath]) # commit and push our changes, if there are any if run(['git', 'diff']): with tempfile.NamedTemporaryFile() as message: message.write(commit_message.encode('utf-8')) message.flush() run(['git', 'commit', '-a', '-F', message.name]) run(['git', 'push', 'origin', 'HEAD']) # switch back before the context manager deletes our cwd os.chdir(origdir)