Extract tr: values from commit instead of database

Instead of loading tracking ids first to the SQL database and then to
the secondary index, extract ids from the commit message. This allows
an administrator to update the tracking id configuration and run just
`java -jar gerrit.war reindex`, without first needing to execute the
older ScanTrackingIds program.

In 2.9 it should be possible to delete ScanTrackingIds.

Change-Id: I56428074461869db6c8dd195cb5a1b689286b114

Documentation/config-gerrit.txt

@@ -2817,9 +2817,13 @@ As example, here is the theme configuration to have the old green look:
 
 Tagged footer lines containing references to external
 tracking systems, parsed out of the commit message and
-saved in Gerrit's database.  After making changes to
-this section, existing changes must be reindexed with the
-link:pgm-ScanTrackingIds.html[ScanTrackingIds] program.
+saved in Gerrit's database.
+
+After making changes to this section, existing changes
+must be reindexed with link:pgm-reindex.html[reindex]
+if index.type is `LUCENE` or `SOLR`; or with
+link:pgm-ScanTrackingIds.html[ScanTrackingIds] if index.type
+is unset or `SQL`.
 
 The tracking ids are searchable using tr:<tracking id> or
 bug:<tracking id>.

Documentation/pgm-ScanTrackingIds.txt

@@ -20,6 +20,14 @@ This task can take quite some time, but can run in the background
 concurrently to the server if the database is MySQL or PostgreSQL.
 If the database is H2, this task must be run by itself.
 
+STATUS
+------
+This command will be replaced by `reindex`.
+
+If secondary indexing is enabled
+(link:config-gerrit.html#index.type[index.type] set to `LUCENE`
+or `SOLR`) use link:pgm-reindex.html[reindex].
+
 OPTIONS
 -------
 

gerrit-server/src/main/java/com/google/gerrit/server/config/TrackingFooters.java

@@ -14,7 +14,13 @@
 
 package com.google.gerrit.server.config;
 
+import com.google.common.collect.Sets;
+
+import org.eclipse.jgit.revwalk.FooterLine;
+
 import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
 
 public class TrackingFooters {
   protected List<TrackingFooter> trackingFooters;
@@ -26,4 +32,24 @@ public class TrackingFooters {
   public List<TrackingFooter> getTrackingFooters() {
     return trackingFooters;
   }
+
+  public Set<String> extract(List<FooterLine> lines) {
+    Set<String> r = Sets.newHashSet();
+    for (FooterLine footer : lines) {
+      for (TrackingFooter config : trackingFooters) {
+        if (footer.matches(config.footerKey())) {
+          Matcher m = config.match().matcher(footer.getValue());
+          while (m.find()) {
+            String id = m.groupCount() > 0
+                ? m.group(1)
+                : m.group();
+            if (!id.isEmpty()) {
+              r.add(id);
+            }
+          }
+        }
+      }
+    }
+    return r;
+  }
 }

gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java

@@ -22,7 +22,6 @@ import com.google.gerrit.reviewdb.client.ChangeMessage;
 import com.google.gerrit.reviewdb.client.PatchLineComment;
 import com.google.gerrit.reviewdb.client.PatchSet;
 import com.google.gerrit.reviewdb.client.PatchSetApproval;
-import com.google.gerrit.reviewdb.client.TrackingId;
 import com.google.gerrit.server.ChangeUtil;
 import com.google.gerrit.server.query.change.ChangeData;
 import com.google.gerrit.server.query.change.ChangeQueryBuilder;
@@ -220,11 +219,12 @@ public class ChangeField {
         @Override
         public Iterable<String> get(ChangeData input, FillArgs args)
             throws OrmException {
-          Set<String> r = Sets.newHashSet();
-          for (TrackingId id : input.trackingIds(args.db)) {
-            r.add(id.getTrackingId());
+          try {
+            return args.trackingFooters.extract(
+                input.commitFooters(args.repoManager, args.db));
+          } catch (IOException e) {
+            throw new OrmException(e);
           }
-          return r;
         }
       };
 

gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java

@@ -15,6 +15,7 @@
 package com.google.gerrit.server.index;
 
 import com.google.gerrit.reviewdb.server.ReviewDb;
+import com.google.gerrit.server.config.TrackingFooters;
 import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.patch.PatchListCache;
 import com.google.gwtorm.server.OrmException;
@@ -59,14 +60,17 @@ public abstract class FieldDef<I, T> {
   public static class FillArgs {
     final Provider<ReviewDb> db;
     final GitRepositoryManager repoManager;
+    final TrackingFooters trackingFooters;
     final PatchListCache patchListCache;
 
     @Inject
     FillArgs(Provider<ReviewDb> db,
         GitRepositoryManager repoManager,
+        TrackingFooters trackingFooters,
         PatchListCache patchListCache) {
       this.db = db;
       this.repoManager = repoManager;
+      this.trackingFooters = trackingFooters;
       this.patchListCache = patchListCache;
     }
   }

gerrit-server/src/main/java/com/google/gerrit/server/query/change/ChangeData.java

@@ -29,7 +29,6 @@ import com.google.gerrit.reviewdb.client.Patch;
 import com.google.gerrit.reviewdb.client.PatchLineComment;
 import com.google.gerrit.reviewdb.client.PatchSet;
 import com.google.gerrit.reviewdb.client.PatchSetApproval;
-import com.google.gerrit.reviewdb.client.Project;
 import com.google.gerrit.reviewdb.client.TrackingId;
 import com.google.gerrit.reviewdb.server.ReviewDb;
 import com.google.gerrit.server.CurrentUser;
@@ -43,8 +42,12 @@ import com.google.gwtorm.server.OrmException;
 import com.google.gwtorm.server.ResultSet;
 import com.google.inject.Provider;
 
+import org.eclipse.jgit.errors.IncorrectObjectTypeException;
+import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.FooterLine;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.RevWalk;
 
@@ -135,6 +138,7 @@ public class ChangeData {
   private ChangeDataSource returnedBySource;
   private Change change;
   private String commitMessage;
+  private List<FooterLine> commitFooters;
   private PatchSet currentPatchSet;
   private Set<PatchSet.Id> limitedIds;
   private Collection<PatchSet> patches;
@@ -318,25 +322,39 @@ public class ChangeData {
   public String commitMessage(GitRepositoryManager repoManager,
       Provider<ReviewDb> db) throws IOException, OrmException {
     if (commitMessage == null) {
-      PatchSet.Id psId = change(db).currentPatchSetId();
-      String sha1 = db.get().patchSets().get(psId).getRevision().get();
-      Project.NameKey name = change.getProject();
-      Repository repo = repoManager.openRepository(name);
-      try {
-        RevWalk walk = new RevWalk(repo);
-        try {
-          RevCommit c = walk.parseCommit(ObjectId.fromString(sha1));
-          commitMessage = c.getFullMessage();
-        } finally {
-          walk.release();
-        }
-      } finally {
-        repo.close();
-      }
+      loadCommitData(repoManager, db);
     }
     return commitMessage;
   }
 
+  public List<FooterLine> commitFooters(GitRepositoryManager repoManager,
+      Provider<ReviewDb> db) throws IOException, OrmException {
+    if (commitFooters == null) {
+      loadCommitData(repoManager, db);
+    }
+    return commitFooters;
+  }
+
+  private void loadCommitData(GitRepositoryManager repoManager,
+      Provider<ReviewDb> db) throws OrmException, RepositoryNotFoundException,
+      IOException, MissingObjectException, IncorrectObjectTypeException {
+    PatchSet.Id psId = change(db).currentPatchSetId();
+    String sha1 = db.get().patchSets().get(psId).getRevision().get();
+    Repository repo = repoManager.openRepository(change.getProject());
+    try {
+      RevWalk walk = new RevWalk(repo);
+      try {
+        RevCommit c = walk.parseCommit(ObjectId.fromString(sha1));
+        commitMessage = c.getFullMessage();
+        commitFooters = c.getFooterLines();
+      } finally {
+        walk.release();
+      }
+    } finally {
+      repo.close();
+    }
+  }
+
   /**
    * @param db review database.
    * @return patches for the change. If {@link #limitToPatchSets(Collection)}