Treat plugin submodule changes like other core commits in release_noter

Improve how plugin changes get written in markdown, in a reliable way.

Simplify the whole script as a consequence. After this change, the
'check' option is no longer needed and will be removed in a next commit.

Bug: Issue 13517
Change-Id: I73ebe8571b583b570d53def613d7b204f8e6eb48

tools/release_noter/release_noter.md.template

@@ -32,3 +32,5 @@ Documentation: **[{{ data.new }}](https://gerrit-documentation.storage.googleapi
 * Projects
 
 ## End-to-end tests
+
+## Plugin changes

tools/release_noter/release_noter.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import argparse
+import os
 import re
 import subprocess
 
@@ -85,9 +86,6 @@ EXCLUDED_SUBJECTS = {
 COMMIT_SHA1_PATTERN = r"^commit ([a-z0-9]+)$"
 DATE_HEADER_PATTERN = r"Date: .+"
 SUBJECT_SUBMODULES_PATTERN = r"^Update git submodules$"
-UPDATE_SUBMODULE_PATTERN = r"\* Update ([a-z/\-]+) from branch '.+'"
-SUBMODULE_SUBJECT_PATTERN = r"^- (.+)"
-SUBMODULE_MERGE_PATTERN = r".+Merge .+"
 ISSUE_ID_PATTERN = r"[a-zA-Z]+: [Ii]ssue ([0-9]+)"
 CHANGE_ID_PATTERN = r"^Change-Id: [I0-9a-z]+$"
 PLUGIN_PATTERN = r"plugins/([a-z\-]+)"
@@ -105,6 +103,8 @@ ISSUE_URL = "https://bugs.chromium.org/p/gerrit/issues/detail?id="
 CHECK_DISCLAIMER = "experimental and much slower"
 MARKDOWN = "release_noter"
 GIT_COMMAND = "git"
+GIT_PATH = "../.."
+PLUGINS = "plugins/"
 UTF8 = "UTF-8"
 
 
@@ -148,7 +148,7 @@ def check_args(options):
     return release_option.group(1)
 
 
-def newly_released(commit_sha1, release):
+def newly_released(commit_sha1, release, cwd):
     if release is None:
         return True
     git_tag = [
@@ -157,7 +157,9 @@ def newly_released(commit_sha1, release):
         "--contains",
         commit_sha1,
     ]
-    process = subprocess.check_output(git_tag, stderr=subprocess.PIPE, encoding=UTF8)
+    process = subprocess.check_output(
+        git_tag, cwd=cwd, stderr=subprocess.PIPE, encoding=UTF8
+    )
     verdict = True
     for line in process.splitlines():
         line = line.strip()
@@ -168,14 +170,25 @@ def newly_released(commit_sha1, release):
     return verdict
 
 
-def open_git_log(options):
+def list_submodules():
+    submodule_names = [
+        GIT_COMMAND,
+        "submodule",
+        "foreach",
+        "--quiet",
+        "echo $name",
+    ]
+    return subprocess.check_output(submodule_names, cwd=f"{GIT_PATH}", encoding=UTF8)
+
+
+def open_git_log(options, cwd=os.getcwd()):
     git_log = [
         GIT_COMMAND,
         "log",
         "--no-merges",
         options.range,
     ]
-    return subprocess.check_output(git_log, encoding=UTF8)
+    return subprocess.check_output(git_log, cwd=cwd, encoding=UTF8)
 
 
 class Component:
@@ -188,6 +201,18 @@ class Component:
 
 
 class Components(Enum):
+    plugin_ce = Component("Codemirror-editor", {PLUGINS})
+    plugin_cm = Component("Commit-message-length-validator", {PLUGINS})
+    plugin_dp = Component("Delete-project", {PLUGINS})
+    plugin_dc = Component("Download-commands", {PLUGINS})
+    plugin_gt = Component("Gitiles", {PLUGINS})
+    plugin_ho = Component("Hooks", {PLUGINS})
+    plugin_pm = Component("Plugin-manager", {PLUGINS})
+    plugin_re = Component("Replication", {PLUGINS})
+    plugin_rn = Component("Reviewnotes", {PLUGINS})
+    plugin_su = Component("Singleusergroup", {PLUGINS})
+    plugin_wh = Component("Webhooks", {PLUGINS})
+
     ui = Component(
         "Polygerrit UI",
         {"poly", "gwt", "button", "dialog", "icon", "hover", "menu", "ux"},
@@ -199,42 +224,29 @@ class Components(Enum):
     otherwise = Component("Other core", {})
 
 
-class Change:
-    subject = None
-    issues = set()
-
-
 class Task(Enum):
     start_commit = 1
     finish_headers = 2
     capture_subject = 3
-    capture_submodule = 4
+    finish_commit = 4
-    capture_submodule_subject = 5
-    finish_submodule_change = 6
-    finish_commit = 7
 
 
 class Commit:
     sha1 = None
     subject = None
-    submodule = None
     issues = set()
 
     def reset(self, signature, task):
         if signature is not None:
             self.sha1 = signature.group(1)
             self.subject = None
-            self.submodule = None
             self.issues = set()
             return Task.finish_headers
         return task
 
 
-def parse_log(process, release, gerrit, options):
+def parse_log(process, release, gerrit, options, commits, cwd=os.getcwd()):
     commit = Commit()
-    commits = init_components()
-    submodules = dict()
-    submodule_change = None
     task = Task.start_commit
     for line in process.splitlines():
         line = line.strip()
@@ -246,32 +258,8 @@ def parse_log(process, release, gerrit, options):
             if re.match(DATE_HEADER_PATTERN, line):
                 task = Task.capture_subject
         elif task == Task.capture_subject:
-            if re.match(SUBJECT_SUBMODULES_PATTERN, line):
-                task = Task.capture_submodule
-            else:
             commit.subject = line
             task = Task.finish_commit
-        elif task == Task.capture_submodule:
-            commit.submodule = re.search(UPDATE_SUBMODULE_PATTERN, line).group(1)
-            if commit.submodule not in submodules:
-                submodules[commit.submodule] = []
-            task = Task.capture_submodule_subject
-        elif task == Task.capture_submodule_subject:
-            submodule_subject = re.search(SUBMODULE_SUBJECT_PATTERN, line)
-            if submodule_subject is not None:
-                if not re.match(SUBMODULE_MERGE_PATTERN, line):
-                    submodule_change = change(submodule_subject, submodules, commit)
-                    task = Task.finish_submodule_change
-            else:
-                task = update_task(line, commit, task)
-        elif task == Task.finish_submodule_change:
-            submodule_issue = re.search(ISSUE_ID_PATTERN, line)
-            if submodule_issue is not None:
-                if submodule_change is not None:
-                    issue_id = submodule_issue.group(1)
-                    submodule_change.issues.add(issue_id)
-            else:
-                task = update_task(line, commit, task)
         elif task == Task.finish_commit:
             commit_issue = re.search(ISSUE_ID_PATTERN, line)
             if commit_issue is not None:
@@ -279,36 +267,15 @@ def parse_log(process, release, gerrit, options):
             else:
                 commit_end = re.match(CHANGE_ID_PATTERN, line)
                 if commit_end is not None:
-                    commit = finish(commit, commits, release, gerrit, options)
+                    commit = finish(commit, commits, release, gerrit, options, cwd)
                     task = Task.start_commit
         else:
             raise RuntimeError("FIXME")
-    return commits, submodules
 
 
-def change(submodule_subject, submodules, commit):
+def finish(commit, commits, release, gerrit, options, cwd):
-    submodule_change = Change()
+    if re.match(SUBJECT_SUBMODULES_PATTERN, commit.subject):
-    submodule_change.subject = submodule_subject.group(1)
+        return Commit()
-    for exclusion in EXCLUDED_SUBJECTS:
-        if exclusion in submodule_change.subject:
-            return None
-    for noted_change in submodules[commit.submodule]:
-        if noted_change.subject == submodule_change.subject:
-            return noted_change
-    escape_these(submodule_change)
-    submodule_change.issues = set()
-    submodules[commit.submodule].append(submodule_change)
-    return submodule_change
-
-
-def update_task(line, commit, task):
-    update_end = re.search(COMMIT_SHA1_PATTERN, line)
-    if update_end is not None:
-        task = commit.reset(update_end, task)
-    return task
-
-
-def finish(commit, commits, release, gerrit, options):
     if len(commit.issues) == 0:
         for exclusion in EXCLUDED_SUBJECTS:
             if exclusion in commit.subject:
@@ -317,22 +284,29 @@ def finish(commit, commits, release, gerrit, options):
             for noted_commit in commits[component]:
                 if noted_commit.subject == commit.subject:
                     return Commit()
-    if newly_released(commit.sha1, release):
+    if newly_released(commit.sha1, release, cwd):
-        set_component(commit, commits)
+        set_component(commit, commits, cwd)
         link_subject(commit, gerrit, options)
         escape_these(commit)
     else:
-        print(f"Previously released: commit {commit.sha1}")
+        prefix = ""
+        if PLUGINS in cwd:
+            prefix = cwd
+        print(f"Previously released: {prefix} commit {commit.sha1}")
     return Commit()
 
 
-def set_component(commit, commits):
+def set_component(commit, commits, cwd):
     component_found = False
     for component in Components:
         for sentinel in component.value.sentinels:
-            if not component_found and sentinel.lower() in commit.subject.lower():
+            if not component_found:
-                commits[component].append(commit)
+                if re.match(f"{GIT_PATH}/{PLUGINS}{component.value.name.lower()}", cwd):
                     component_found = True
+                elif sentinel.lower() in commit.subject.lower():
+                    component_found = True
+                if component_found:
+                    commits[component].append(commit)
     if not component_found:
         commits[Components.otherwise].append(commit)
 
@@ -363,20 +337,15 @@ def escape_these(in_change):
 
 def print_commits(commits, md):
     for component in commits:
+        if len(commits[component]) > 0:
+            if PLUGINS in component.value.sentinels:
+                md.write(f"\n### {component.value.name}\n")
+            else:
                 md.write(f"\n## {component.value.name} changes\n")
             for commit in commits[component]:
                 print_from(commit, md)
 
 
-def print_submodules(submodules, md):
-    md.write("\n## Plugin changes\n")
-    for submodule in sorted(submodules):
-        plugin = re.search(PLUGIN_PATTERN, submodule)
-        md.write(f"\n### {plugin.group(1)}\n")
-        for submodule_change in submodules[submodule]:
-            print_from(submodule_change, md)
-
-
 def print_from(this_change, md):
     md.write("\n*")
     for issue in sorted(this_change.issues):
@@ -401,7 +370,7 @@ def print_template(md, options):
     md.write(f"{template.render(data=data)}\n")
 
 
-def print_notes(commits, submodules, options):
+def print_notes(commits, options):
     markdown = f"{MARKDOWN}.md"
     next_md = 2
     while path.exists(markdown):
@@ -409,17 +378,33 @@ def print_notes(commits, submodules, options):
         next_md += 1
     with open(markdown, "w") as md:
         print_template(md, options)
-        print_submodules(submodules, md)
         print_commits(commits, md)
         md.write("\n## Bugfix releases\n")
 
 
+def plugin_changes():
+    plugin_commits = init_components()
+    for submodule_name in list_submodules().splitlines():
+        plugin_name = re.search(PLUGIN_PATTERN, submodule_name)
+        if plugin_name is not None:
+            plugin_wd = f"{GIT_PATH}/{PLUGINS}{plugin_name.group(1)}"
+            plugin_log = open_git_log(script_options, plugin_wd)
+            parse_log(
+                plugin_log,
+                release_tag,
+                gerrit_api,
+                script_options,
+                plugin_commits,
+                plugin_wd,
+            )
+    return plugin_commits
+
+
 if __name__ == "__main__":
     gerrit_api = GerritRestAPI(url=GERRIT_URL, auth=Anonymous())
     script_options = parse_args()
     release_tag = check_args(script_options)
+    noted_changes = plugin_changes()
     change_log = open_git_log(script_options)
-    core_changes, submodule_changes = parse_log(
+    parse_log(change_log, release_tag, gerrit_api, script_options, noted_changes)
-        change_log, release_tag, gerrit_api, script_options
+    print_notes(noted_changes, script_options)
-    )
-    print_notes(core_changes, submodule_changes, script_options)