starlingx/root
Code Issues Proposed changes

merge-topic.py

Browse Source 
A tool to merge all gerrit reviews based on topic.

merge-topic.py repo --help

usage: merge-topic.py repo [-h]
         --topic TOPIC [--repo-root-dir REPO_ROOT_DIR]
         [--manifest MANIFEST]
         [--download-strategy {Pull,Cherry Pick,Branch,Checkout}]
         [--status {open,merged,abandoned}]
         [--merge-fixer MERGE_FIXER]
         [--avoid-re-download]
         [--dry-run]

optional arguments:
  -h, --help    show this help message and exit
  --topic TOPIC, -t TOPIC
                Gerrit topic... can be specified more than once
                (default: None)
  --repo-root-dir REPO_ROOT_DIR, -rr REPO_ROOT_DIR
                Path to repo root dir (default:
                /localdisk/designer/slittle1/WRCP_DEV)
  --manifest MANIFEST, -m MANIFEST
                File name of the manifest file (not path).  Otherwise
                use the manifest selected by the last "repo init"
                (default: None)
  --download-strategy {Pull,Cherry Pick,Branch,Checkout}
                Strategy to download the patch: Pull, Cherry Pick,
                Branch, Checkout (default: Cherry Pick)
  --status {open,merged,abandoned}, -s {open,merged,abandoned}
                Status of the review... can be specified more than
                once (default: ['open'])
  --merge-fixer MERGE_FIXER, -mf MERGE_FIXER
                Script to be run to attempt auto merge fixing, e.g.
                pick_both_merge_fixer.py (default: None)
  --avoid-re-download, -ard
                Avoid re-downloading a commit if it already exists in
                the git repo. (default: False)
  --dry-run     Simulate, but don't sync (default: False)

Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: I2a11af26368fd10d9457ddb39101c7e136f478c9
This commit is contained in:
Scott Little
2025-05-15 15:57:34 -04:00
parent 1f30f20a49
commit d10d5361e8
3 changed files with 643 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
build-tools/merge-topic/README.md Normal file
View File

@@ -0,0 +1,96 @@
# gerrit-topic-picker
## Features for repositories managed by repo tool
- Can filter status
merge-topic.py --status/-s open --status/-s merged .. --status/-s whatever
- Can avoid re-downloading a review if a commit with the Change-Id is present
merge-topic.py --avoid-re-download/-ard
- Specify one or more topics
merge-topic.py --topic target-topic-1 --topic target-topic-2
merge-topic.py -t target-topic
- gerrit URL is automatically discovered from the .gitreview file for each repo
- Specify download strategy
merge-topic.py --download-strategy/-ds "Cherry Pick"
merge-topic.py --download-strategy/-ds "Pull"
- Specify script to be run in case of merge conflict. A simple fixer is supplied by "pick_both_merge_fixer.py".
Otherwise provide your own.
merge-topic.py --merge-fixer pick_both_merge_fixer.py
merge-topic.py --merge-fixer my_script.sh
merge-topic.py --merge-fixer my_script.py
merge-topic.py --merge-fixer my_script.runnable
## Usage for repositories managed by repo tool:
Set the `MY_REPO_ROOT_DIR` environment variable to the repo root directory (the one which contains the `.repo` dir)
MY_REPO_ROOT_DIR=/path/to/repo/ python3 merge-topic.py repo --help
Example usage in real life:
MY_REPO_ROOT_DIR=/here \
python3 merge-topic.py repo \
--topic my-topic \
--download-strategy "Cherry Pick" \
--status open \
--avoid-re-download
# OR short
MY_REPO_ROOT_DIR=/here \
python3 merge-topic.py repo \
-t my-topic \
-ds "Cherry Pick" \
-s open \
-ard
# fails a cherry-pick: CalledProcessError(1, ' git cherry-pick FETCH_HEAD')
# resolve the cherry-pick merge errors
# then invoke the tool againg with the same parameters
# repeat the process until all commits are synced
Example usage for specifying a script that could automatically fix merge conflicts:
MY_REPO_ROOT_DIR=/here \
python3 merge-topic.py repo \
--topic my-topic \
--download-strategy "Cherry Pick" \
--status open \
--avoid-re-download \
--merge-fixer dummy_merge_fixer.py
Example real life usage with merge fixer that picks changes from both sources:
MY_REPO_ROOT_DIR=/here \
python3 merge-topic.py repo \
--topic my-topic \
--download-strategy "Cherry Pick" \
--status open \
--avoid-re-download \
--merge-fixer pick_both_merge_fixer.py
Example usage for syncing open and merged reviews:
MY_REPO_ROOT_DIR=/here \
python3 merge-topic.py repo \
--topic my-topic \
--download-strategy "Cherry Pick" \
--status open \
--status merged \
--avoid-re-download
## Future Work
- Pick relation chain
- Improve merge fixer logging
- Fully automate merge fixer

464
build-tools/merge-topic/merge-topic.py Executable file
View File

@@ -0,0 +1,464 @@
#!/usr/bin/python3
#
# Copyright (c) 2021,2025 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import argparse
import configparser
from datetime import datetime
import json
import os
import pprint
import subprocess
import sys
import urllib.parse
import xml.etree.ElementTree as ET
import requests
SUCCESS = 0
FAILURE = 1
DEFAULT_MANIFEST_FILE='default.xml'
# Cache the repo manifest file information
REPO_MANIFEST = None
def responseCorrection(content):
return content[5:]
def handleList(dargs):
pass
def etree_to_dict(elem):
""" Convert in elementTree to a dictionary
"""
d = {elem.tag: {} if elem.attrib else None}
# Handle element's children
children = list(elem)
if children:
dd = {}
for child in children:
child_dict = etree_to_dict(child)
for k, v in child_dict.items():
if k in dd:
if not isinstance(dd[k], list):
dd[k] = [dd[k]]
dd[k].append(v)
else:
dd[k] = v
d[elem.tag] = dd
# Handle element's attributes
if elem.attrib:
d[elem.tag].update((k, v) for k, v in elem.attrib.items())
# Handle element's text
text = elem.text.strip() if elem.text and elem.text.strip() else None
if text:
d[elem.tag]['#text'] = text
return d
def addGerritQuery(query_string, field_name, target_values):
""" Add a query for a specific field.
"""
if not type(target_values) is list:
target_values = [target_values]
if len(target_values) == 0:
return query_string
elif len(target_values) == 1:
return '{} {}:"{}"'.format(query_string, field_name, target_values[0])
else:
assemble = '{} ({}:"{}"'.format(query_string, field_name, target_values[0])
for val in target_values[1:]:
assemble = '{} OR {}:"{}"'.format(assemble, field_name, val)
assemble = assemble + ')'
return assemble
def gerritQuery(query):
url_query = urllib.parse.urljoin('https://' + query['gerrit'], 'changes/')
if query['verbose'] >= 1:
print('gerritQuery args: {}'.format(url_query))
# GET /changes/?q=topic:"my-topic"&o=CURRENT_REVISION&o=DOWNLOAD_COMMANDS HTTP/1.0
# GET /changes/?q=topic:"my-topic"+status:open&o=CURRENT_REVISION&o=DOWNLOAD_COMMANDS HTTP/1.0
# GET /changes/?q=topic:"my-topic"+(status:open OR status:merged)&o=CURRENT_REVISION&o=DOWNLOAD_COMMANDS HTTP/1.0
query_string = addGerritQuery('', 'topic', query['topic'])
query_string = addGerritQuery(query_string, 'status', query['status'])
query_string = addGerritQuery(query_string, 'branch', query['branch'])
if 'repo' in query:
repo = query['repo']
if repo.endswith('.git'):
repo = repo[:-len('.git')]
query_string = addGerritQuery(query_string, 'repo', repo)
if query['verbose'] >= 1:
print('gerritQuery string: {}'.format(query_string))
params = {'q': query_string,
'o': ['CURRENT_REVISION', 'DOWNLOAD_COMMANDS']}
r = requests.get(url=url_query, params=params)
content = responseCorrection(r.text)
data = json.loads(content)
if query['verbose'] >= 5:
print('gerritQuery results:')
pprint.pprint(data)
sorted_data = sorted(data, key=lambda x: truncate_ns_to_us(x["updated"]))
if query['verbose'] >= 4:
print('gerritQuery results:')
pprint.pprint(sorted_data)
return sorted_data
def truncate_ns_to_us(ts: str) -> datetime:
if '.' in ts:
base, frac = ts.split('.')
frac = (frac + '000000')[:6] # pad and truncate to 6 digits
ts = f"{base}.{frac}"
return datetime.strptime(ts, "%Y-%m-%d %H:%M:%S.%f")
def readRepoManifest(repo_root_dir, manifest_file=None):
if manifest_file:
manifest_path = os.path.join(repo_root_dir, '.repo', 'manifests', manifest_file)
else:
manifest_path = os.path.join(repo_root_dir, '.repo', 'manifest.xml')
print('Reading manifest file: {}'.format(manifest_path))
tree = ET.parse(manifest_path)
root = tree.getroot()
manifest = {}
manifest['root'] = repo_root_dir
manifest['file_name'] = manifest_file
manifest['path'] = manifest_path
manifest['remote'] = {}
manifest['project'] = {}
manifest['default'] = {}
for element in root.findall('remote'):
if dargs['verbose'] >= 5:
print(element.tag, element.attrib)
remote_name = element.get('name')
fetch_url = element.get('fetch')
push_url = element.get('pushurl')
review = element.get('review')
revision = element.get('revision')
manifest['remote'][remote_name]={'fetch': fetch_url, 'push_url': push_url, 'review': review, 'revision': revision}
for element in root.findall('project'):
if dargs['verbose'] >= 5:
print(element.tag, element.attrib)
project_name = element.get('name')
remote_name = element.get('remote')
path = element.get('path')
revision = element.get('revision')
groups = element.get('groups')
manifest['project'][project_name]={'remote': remote_name, 'path': path, 'revision': revision, 'groups': groups}
for element in root.findall('default'):
remote_name = element.get('remote')
revision = element.get('revision')
manifest['default']['remote'] = remote_name
manifest['default']['revision'] = revision
for element in root.findall('include'):
include_name = element.get('name')
include_manifest = readRepoManifest(repo_root_dir, include_name)
manifest = {**manifest, **include_manifest}
return manifest
def RepoManifestProjectList(manifest):
return list(manifest['project'].keys())
def RepoManifestRemoteList(manifest):
return list(manifest['remote'].keys())
def RepoManifestProjectInfo(manifest, project_name, use_defaults=True, abs_path=True):
project_info = None
if project_name not in manifest['project']:
return None
project_info = manifest['project'][project_name]
if use_defaults:
if project_info['remote'] is None:
project_info['remote'] = manifest['default']['remote']
if project_info['revision'] is None:
if project_info['remote'] is not None:
project_info['revision'] = manifest['remote'][project_info['remote']]['revision']
if project_info['revision'] is None:
project_info['revision'] = manifest['default']['revision']
if abs_path:
project_info['path'] = os.path.join(manifest['root'], project_info['path'])
return project_info
def RepoManifestProjectPath(manifest, project_name, abs_path=True):
project_info = RepoManifestProjectInfo(manifest, project_name, abs_path=abs_path)
if project_info is None:
return None
return project_info['path']
def extractDownloadCommand(dargs, change):
rev = change.get('revisions')
key = list(rev.keys())[0]
command = rev.get(key)
command = command.get('fetch')
command = command.get('anonymous http')
command = command.get('commands')
command = command.get(dargs['download_strategy'], None)
if not command:
raise Exception("Can't get command for {} download strategy!".format(
dargs['download_strategy']))
return command
def checkSkipChange(dargs, change_id, max_search_depth=100):
""" Determine if the change should be skipped.
Determine based on the Change-Id: in commit message.
@param dargs: Parsed dargs
@param change_id: A gerrit Change-Id to be skipped
@param max_search_depth: Limit the search depth to a certain number
to speed up things.
@return: True if the change should be skipped
"""
cmd = ['git', 'rev-list', 'HEAD', '--count', '--no-merges']
output = subprocess.check_output(
cmd
, errors="strict").strip()
rev_count = int(output)
if dargs['verbose']>= 6:
print(rev_count)
# TODO param for max_search_depth
for i in range(min(rev_count - 1, max_search_depth)):
cmd = ['git', 'rev-list', '--format=%B', '--max-count',
'1', 'HEAD~{}'.format(i)]
output = subprocess.check_output(
cmd
, errors="strict").strip()
if dargs['verbose']>= 6:
print(output)
# TODO avoid false positives, search just last occurrence
if 'Change-Id: {}'.format(change_id) in output:
print('Found {} in git log'.format(change_id))
return True
return False
def validateHandleRepoArgs(dargs):
""" Validate dargs for repositories that use Repo tool
@param dargs: Args from ArgumentParser
"""
print('Using repo root dir {}'.format(dargs['repo_root_dir']))
if not os.path.exists(dargs['repo_root_dir']):
print('{} does not exist'.format(dargs['repo_root_dir']))
return False
print('Using manifest {}'.format(dargs['manifest']))
manifest_path = getRepoManifestPath(dargs['repo_root_dir'], dargs['manifest'])
if not os.path.exists(manifest_path):
print('{} does not exist'.format(manifest_path))
return False
# print('Using gerrit {}'.format(dargs['gerrit']))
print('Using download strategy {}'.format(dargs['download_strategy']))
# print('Using review statuses {}'.format(dargs['status']))
if dargs['merge_fixer']:
if os.path.exists(dargs['merge_fixer']):
print('Using script to attempt automatic merge conflicts '
'resolution: {}'.format(dargs['merge_fixer']))
else:
print('File {} does not exist'.format(dargs['merge_fixer']))
return False
return True
def handleRepo(args):
""" Main logic for repositories that use repo tool
@param args: Args from ArgumentParser
"""
global REPO_MANIFEST
dargs = vars(args)
validateHandleRepoArgs(dargs)
tool_cwd = os.getcwd()
REPO_MANIFEST = readRepoManifest(dargs['repo_root_dir'])
for project in RepoManifestProjectList(REPO_MANIFEST):
project_info = RepoManifestProjectInfo(REPO_MANIFEST, project)
project_path = project_info['path']
print(project_info)
git_review_info = readGitReview(project_info['path'])
if git_review_info is None:
print('Skipping {}: .gitreview missing'.format(project))
continue
if git_review_info['branch'] != project_info['revision']:
print('Skipping {}: branch mismatch between manifest and gitreview'.format(project))
continue
query = {}
query['gerrit'] = git_review_info['host']
query['repo'] = git_review_info['project']
query['branch'] = git_review_info['branch']
query['topic'] = dargs['topic']
query['status'] = 'open'
query['verbose'] = dargs['verbose']
if dargs['verbose']>= 1:
print(project, ' ', query)
query_results = gerritQuery(query)
print('Found {} matching reviews in {}'.format(len(query_results),project))
for query_result in query_results:
# Get project of the change
project = query_result.get('project')
project_name, repository_name = project.split('/')
change_id = query_result.get('change_id')
print('Detected change number {} ID {} project {} repository {}'
''.format(query_result.get('_number', ''),
change_id,
project_name,
repository_name))
download_command = extractDownloadCommand(dargs, query_result)
os.chdir(project_path)
print("Changed working directory to: {}".format(os.getcwd()))
# Check if the change should be skipped
if dargs['avoid_re_download'] and checkSkipChange(dargs, change_id):
print('Skipping {}'.format(change_id))
continue
# Apply commit
cmds = download_command.split('&&')
print('Commands to be executed {}'.format(cmds))
try:
oldenv = os.environ.copy()
env={'GIT_RERERE_AUTOUPDATE': '0'}
env = { **oldenv, **env }
for cmd in list(cmds):
cmd = cmd.strip('"')
print('Command to be executed {}'.format(cmd))
if not dargs['dry_run']:
output = subprocess.check_output(
cmd
, env
, errors="strict", shell=True).strip()
print('Executed: \n{}'.format(output))
except Exception as e:
pprint.pprint(e)
if dargs['merge_fixer'] and not dargs['dry_run']:
print('Using merge fixer!')
runMergeFixer(dargs, project_path, tool_cwd)
else:
return False
return True
def runMergeFixer(dargs, project_path, tool_cwd):
# Run fixer
fixer = '{}'.format(os.path.join(tool_cwd, dargs['merge_fixer']))
cmd = [fixer]
fixer_rc, _ = run_cmd(cmd, shell=False, halt_on_exception=False)
# Abort in case of fixer run failure
if fixer_rc == FAILURE:
print('Fixer failed, aborting!!!')
return False
def run_cmd(cmd, shell=False, halt_on_exception=False):
# TODO improve logging, but not worth at the moment.
# LIMITATION
# Now we could automate up to doing the cherry-pick continue.
# `git cherry-pick --continue` opens a text editor and freezes terminal
# Need to figure a way to go around that.
try:
print('Running {}:\n'.format(cmd))
p1 = subprocess.Popen(
cmd,
errors="strict",
shell=shell,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
output, error = p1.communicate()
print('{}\n'.format(output))
found_exception = False
if p1.returncode != 0:
print('stderr: {}\n'.format(error))
found_exception = True
except Exception as e:
found_exception = True
pprint.pprint(e)
finally:
if found_exception and halt_on_exception:
exit(1)
if not found_exception:
return SUCCESS, output
return FAILURE, output
def readGitReview(gitRoot):
result = {}
config = configparser.ConfigParser()
git_review_path = os.path.join(gitRoot, '.gitreview')
if not os.path.exists(git_review_path):
return None
config.read(os.path.join(gitRoot, '.gitreview'))
result['host'] = config['gerrit']['host']
result['port'] = int(config['gerrit']['port'])
result['project'] = config['gerrit']['project']
result['branch'] = config['gerrit'].get('defaultbranch', 'master')
return result
def main():
parser = argparse.ArgumentParser(description='Tool to sync a Gerrit topic(s)',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
epilog='''Use %(prog)s subcommand --help to get help for all of parameters''')
parser.add_argument('--verbose', '-v', action='count', default=0, help='Verbosity level')
subparsers = parser.add_subparsers(title='Repository type control Commands',
help='...')
# TODO GIT
repo_parser = subparsers.add_parser('git',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='Command for handling a git managed project... not supported yet')
repo_parser.set_defaults(handle=handleList)
# REPO
repo_parser = subparsers.add_parser('repo',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
help='Command for handling a repo managed project')
repo_parser.add_argument('--topic', '-t',
action='append',
help='Gerrit topic... can be specified more than once',
required=True)
repo_parser.add_argument('--repo-root-dir', '-rr',
help='Path to repo root dir',
default=os.getenv('MY_REPO_ROOT_DIR', os.getcwd()),
required=False)
repo_parser.add_argument('--manifest', '-m',
help='File name of the manifest file (not path). Otherwise use the manifest selected by the last "repo init"',
default=None,
required=False)
repo_parser.add_argument('--download-strategy', '-ds',
help='Strategy to download the patch: Pull, Cherry Pick, Branch, Checkout',
choices=['Pull', 'Cherry Pick', 'Branch', 'Checkout'],
default='Cherry Pick',
required=False)
repo_parser.add_argument('--status', '-s',
action='append',
help='Status of the review... can be specified more than once',
choices=['open', 'merged', 'abandoned'],
default=['open'],
required=False)
repo_parser.add_argument('--merge-fixer', '-mf',
help='Script to be run to attempt auto merge fixing, e.g. pick_both_merge_fixer.py',
required=False)
repo_parser.add_argument('--avoid-re-download', '-ard',
action='store_true',
help='Avoid re-downloading a commit if it already exists in the git repo.',
default=False,
required=False)
repo_parser.add_argument('--dry-run',
action='store_true',
help='''Simulate, but don't sync''',
default=False,
required=False)
repo_parser.set_defaults(handle=handleRepo)
args = parser.parse_args()
if hasattr(args, 'handle'):
args.handle(args)
else:
parser.print_help()
if __name__ == '__main__':
main()

83
build-tools/merge-topic/pick_both_merge_fixer.py Normal file
View File

@@ -0,0 +1,83 @@
#!/usr/bin/python3
#
# Copyright (c) 2021,2025 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import os
import pprint
import subprocess
CHERRY_PICKING = 'You are currently cherry-picking commit'
BOTH_ADDED = 'both added:'
BOTH_MODIFIED = 'both modified:'
SUCCESS = 0
FAILURE = 1
def run_cmd(cmd, env=None, shell=False, halt_on_exception=False):
try:
print('Running {} with env {}:\n'.format(cmd, env))
oldenv = os.environ.copy()
# env = oldenv | env
env = { **oldenv, **env }
output = subprocess.check_output(
cmd
, env=env
, errors="strict"
, shell=shell).strip()
found_exception = False
except Exception as e:
found_exception = True
pprint.pprint(e)
finally:
if found_exception and halt_on_exception:
exit(1)
if not found_exception:
return SUCCESS, output
return FAILURE, None
print('CWD {}'.format(os.getcwd()))
rc, out = run_cmd(['git', 'status'])
if rc != SUCCESS:
exit(rc)
if CHERRY_PICKING in out:
print('Detected cherry-picking {}'.format(os.getcwd()))
for status_line in out.splitlines():
if BOTH_ADDED in status_line or BOTH_MODIFIED in status_line:
# Get file
conflict_file = status_line.split(':')[1].strip()
print('Identified conflict file {}'.format(conflict_file))
with open(conflict_file, "r+") as f:
# Need to buffer, so file pointer is reset
buffer = f.readlines()
f.seek(0)
for code_line in buffer:
# TODO better matching
if not code_line.startswith('<<<<<<< ') and \
not (code_line.startswith('=======')) and \
not code_line.startswith('>>>>>>> '):
f.write(code_line)
else:
print('Dropping :{}'.format(code_line))
f.truncate()
# Git add file
rc, out = run_cmd(['git', 'add', conflict_file])
if rc != SUCCESS:
exit(rc)
# Don't call this, it will freeze the terminal
# Git cherry-pick --continue
rc, out = run_cmd(['git', 'cherry-pick', '--continue'], env={'GIT_EDITOR': 'true'})
if rc != SUCCESS:
exit(rc)