From 0288cba71f64d1701dad27fac20d2e37d11a483f Mon Sep 17 00:00:00 2001 From: Marco Miller Date: Mon, 26 Oct 2020 16:11:42 -0400 Subject: [PATCH] Treat plugin submodule changes like other core commits in release_noter Improve how plugin changes get written in markdown, in a reliable way. Simplify the whole script as a consequence. After this change, the 'check' option is no longer needed and will be removed in a next commit. Bug: Issue 13517 Change-Id: I73ebe8571b583b570d53def613d7b204f8e6eb48 --- tools/release_noter/release_noter.md.template | 2 + tools/release_noter/release_noter.py | 181 ++++++++---------- 2 files changed, 85 insertions(+), 98 deletions(-) diff --git a/tools/release_noter/release_noter.md.template b/tools/release_noter/release_noter.md.template index 5cd7373db0..06399a12f3 100644 --- a/tools/release_noter/release_noter.md.template +++ b/tools/release_noter/release_noter.md.template @@ -32,3 +32,5 @@ Documentation: **[{{ data.new }}](https://gerrit-documentation.storage.googleapi * Projects ## End-to-end tests + +## Plugin changes diff --git a/tools/release_noter/release_noter.py b/tools/release_noter/release_noter.py index 54bd916b2e..c3b49214d6 100644 --- a/tools/release_noter/release_noter.py +++ b/tools/release_noter/release_noter.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import argparse +import os import re import subprocess @@ -85,9 +86,6 @@ EXCLUDED_SUBJECTS = { COMMIT_SHA1_PATTERN = r"^commit ([a-z0-9]+)$" DATE_HEADER_PATTERN = r"Date: .+" SUBJECT_SUBMODULES_PATTERN = r"^Update git submodules$" -UPDATE_SUBMODULE_PATTERN = r"\* Update ([a-z/\-]+) from branch '.+'" -SUBMODULE_SUBJECT_PATTERN = r"^- (.+)" -SUBMODULE_MERGE_PATTERN = r".+Merge .+" ISSUE_ID_PATTERN = r"[a-zA-Z]+: [Ii]ssue ([0-9]+)" CHANGE_ID_PATTERN = r"^Change-Id: [I0-9a-z]+$" PLUGIN_PATTERN = r"plugins/([a-z\-]+)" @@ -105,6 +103,8 @@ ISSUE_URL = "https://bugs.chromium.org/p/gerrit/issues/detail?id=" CHECK_DISCLAIMER = "experimental and much slower" MARKDOWN = "release_noter" GIT_COMMAND = "git" +GIT_PATH = "../.." +PLUGINS = "plugins/" UTF8 = "UTF-8" @@ -148,7 +148,7 @@ def check_args(options): return release_option.group(1) -def newly_released(commit_sha1, release): +def newly_released(commit_sha1, release, cwd): if release is None: return True git_tag = [ @@ -157,7 +157,9 @@ def newly_released(commit_sha1, release): "--contains", commit_sha1, ] - process = subprocess.check_output(git_tag, stderr=subprocess.PIPE, encoding=UTF8) + process = subprocess.check_output( + git_tag, cwd=cwd, stderr=subprocess.PIPE, encoding=UTF8 + ) verdict = True for line in process.splitlines(): line = line.strip() @@ -168,14 +170,25 @@ def newly_released(commit_sha1, release): return verdict -def open_git_log(options): +def list_submodules(): + submodule_names = [ + GIT_COMMAND, + "submodule", + "foreach", + "--quiet", + "echo $name", + ] + return subprocess.check_output(submodule_names, cwd=f"{GIT_PATH}", encoding=UTF8) + + +def open_git_log(options, cwd=os.getcwd()): git_log = [ GIT_COMMAND, "log", "--no-merges", options.range, ] - return subprocess.check_output(git_log, encoding=UTF8) + return subprocess.check_output(git_log, cwd=cwd, encoding=UTF8) class Component: @@ -188,6 +201,18 @@ class Component: class Components(Enum): + plugin_ce = Component("Codemirror-editor", {PLUGINS}) + plugin_cm = Component("Commit-message-length-validator", {PLUGINS}) + plugin_dp = Component("Delete-project", {PLUGINS}) + plugin_dc = Component("Download-commands", {PLUGINS}) + plugin_gt = Component("Gitiles", {PLUGINS}) + plugin_ho = Component("Hooks", {PLUGINS}) + plugin_pm = Component("Plugin-manager", {PLUGINS}) + plugin_re = Component("Replication", {PLUGINS}) + plugin_rn = Component("Reviewnotes", {PLUGINS}) + plugin_su = Component("Singleusergroup", {PLUGINS}) + plugin_wh = Component("Webhooks", {PLUGINS}) + ui = Component( "Polygerrit UI", {"poly", "gwt", "button", "dialog", "icon", "hover", "menu", "ux"}, @@ -199,42 +224,29 @@ class Components(Enum): otherwise = Component("Other core", {}) -class Change: - subject = None - issues = set() - - class Task(Enum): start_commit = 1 finish_headers = 2 capture_subject = 3 - capture_submodule = 4 - capture_submodule_subject = 5 - finish_submodule_change = 6 - finish_commit = 7 + finish_commit = 4 class Commit: sha1 = None subject = None - submodule = None issues = set() def reset(self, signature, task): if signature is not None: self.sha1 = signature.group(1) self.subject = None - self.submodule = None self.issues = set() return Task.finish_headers return task -def parse_log(process, release, gerrit, options): +def parse_log(process, release, gerrit, options, commits, cwd=os.getcwd()): commit = Commit() - commits = init_components() - submodules = dict() - submodule_change = None task = Task.start_commit for line in process.splitlines(): line = line.strip() @@ -246,32 +258,8 @@ def parse_log(process, release, gerrit, options): if re.match(DATE_HEADER_PATTERN, line): task = Task.capture_subject elif task == Task.capture_subject: - if re.match(SUBJECT_SUBMODULES_PATTERN, line): - task = Task.capture_submodule - else: - commit.subject = line - task = Task.finish_commit - elif task == Task.capture_submodule: - commit.submodule = re.search(UPDATE_SUBMODULE_PATTERN, line).group(1) - if commit.submodule not in submodules: - submodules[commit.submodule] = [] - task = Task.capture_submodule_subject - elif task == Task.capture_submodule_subject: - submodule_subject = re.search(SUBMODULE_SUBJECT_PATTERN, line) - if submodule_subject is not None: - if not re.match(SUBMODULE_MERGE_PATTERN, line): - submodule_change = change(submodule_subject, submodules, commit) - task = Task.finish_submodule_change - else: - task = update_task(line, commit, task) - elif task == Task.finish_submodule_change: - submodule_issue = re.search(ISSUE_ID_PATTERN, line) - if submodule_issue is not None: - if submodule_change is not None: - issue_id = submodule_issue.group(1) - submodule_change.issues.add(issue_id) - else: - task = update_task(line, commit, task) + commit.subject = line + task = Task.finish_commit elif task == Task.finish_commit: commit_issue = re.search(ISSUE_ID_PATTERN, line) if commit_issue is not None: @@ -279,36 +267,15 @@ def parse_log(process, release, gerrit, options): else: commit_end = re.match(CHANGE_ID_PATTERN, line) if commit_end is not None: - commit = finish(commit, commits, release, gerrit, options) + commit = finish(commit, commits, release, gerrit, options, cwd) task = Task.start_commit else: raise RuntimeError("FIXME") - return commits, submodules -def change(submodule_subject, submodules, commit): - submodule_change = Change() - submodule_change.subject = submodule_subject.group(1) - for exclusion in EXCLUDED_SUBJECTS: - if exclusion in submodule_change.subject: - return None - for noted_change in submodules[commit.submodule]: - if noted_change.subject == submodule_change.subject: - return noted_change - escape_these(submodule_change) - submodule_change.issues = set() - submodules[commit.submodule].append(submodule_change) - return submodule_change - - -def update_task(line, commit, task): - update_end = re.search(COMMIT_SHA1_PATTERN, line) - if update_end is not None: - task = commit.reset(update_end, task) - return task - - -def finish(commit, commits, release, gerrit, options): +def finish(commit, commits, release, gerrit, options, cwd): + if re.match(SUBJECT_SUBMODULES_PATTERN, commit.subject): + return Commit() if len(commit.issues) == 0: for exclusion in EXCLUDED_SUBJECTS: if exclusion in commit.subject: @@ -317,22 +284,29 @@ def finish(commit, commits, release, gerrit, options): for noted_commit in commits[component]: if noted_commit.subject == commit.subject: return Commit() - if newly_released(commit.sha1, release): - set_component(commit, commits) + if newly_released(commit.sha1, release, cwd): + set_component(commit, commits, cwd) link_subject(commit, gerrit, options) escape_these(commit) else: - print(f"Previously released: commit {commit.sha1}") + prefix = "" + if PLUGINS in cwd: + prefix = cwd + print(f"Previously released: {prefix} commit {commit.sha1}") return Commit() -def set_component(commit, commits): +def set_component(commit, commits, cwd): component_found = False for component in Components: for sentinel in component.value.sentinels: - if not component_found and sentinel.lower() in commit.subject.lower(): - commits[component].append(commit) - component_found = True + if not component_found: + if re.match(f"{GIT_PATH}/{PLUGINS}{component.value.name.lower()}", cwd): + component_found = True + elif sentinel.lower() in commit.subject.lower(): + component_found = True + if component_found: + commits[component].append(commit) if not component_found: commits[Components.otherwise].append(commit) @@ -363,18 +337,13 @@ def escape_these(in_change): def print_commits(commits, md): for component in commits: - md.write(f"\n## {component.value.name} changes\n") - for commit in commits[component]: - print_from(commit, md) - - -def print_submodules(submodules, md): - md.write("\n## Plugin changes\n") - for submodule in sorted(submodules): - plugin = re.search(PLUGIN_PATTERN, submodule) - md.write(f"\n### {plugin.group(1)}\n") - for submodule_change in submodules[submodule]: - print_from(submodule_change, md) + if len(commits[component]) > 0: + if PLUGINS in component.value.sentinels: + md.write(f"\n### {component.value.name}\n") + else: + md.write(f"\n## {component.value.name} changes\n") + for commit in commits[component]: + print_from(commit, md) def print_from(this_change, md): @@ -401,7 +370,7 @@ def print_template(md, options): md.write(f"{template.render(data=data)}\n") -def print_notes(commits, submodules, options): +def print_notes(commits, options): markdown = f"{MARKDOWN}.md" next_md = 2 while path.exists(markdown): @@ -409,17 +378,33 @@ def print_notes(commits, submodules, options): next_md += 1 with open(markdown, "w") as md: print_template(md, options) - print_submodules(submodules, md) print_commits(commits, md) md.write("\n## Bugfix releases\n") +def plugin_changes(): + plugin_commits = init_components() + for submodule_name in list_submodules().splitlines(): + plugin_name = re.search(PLUGIN_PATTERN, submodule_name) + if plugin_name is not None: + plugin_wd = f"{GIT_PATH}/{PLUGINS}{plugin_name.group(1)}" + plugin_log = open_git_log(script_options, plugin_wd) + parse_log( + plugin_log, + release_tag, + gerrit_api, + script_options, + plugin_commits, + plugin_wd, + ) + return plugin_commits + + if __name__ == "__main__": gerrit_api = GerritRestAPI(url=GERRIT_URL, auth=Anonymous()) script_options = parse_args() release_tag = check_args(script_options) + noted_changes = plugin_changes() change_log = open_git_log(script_options) - core_changes, submodule_changes = parse_log( - change_log, release_tag, gerrit_api, script_options - ) - print_notes(core_changes, submodule_changes, script_options) + parse_log(change_log, release_tag, gerrit_api, script_options, noted_changes) + print_notes(noted_changes, script_options)