diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index ccaabc706a..814d32ec20 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt @@ -1560,9 +1560,11 @@ By default a shade of yellow, `FFFFCC`. [[trackingid]] Section trackingid ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Tagged footer lines containing references to external tracking -systems, parsed out of the commit message and saved in Gerrit's -database. +Tagged footer lines containing references to external +tracking systems, parsed out of the commit message and +saved in Gerrit's database. After making changes to +this section, existing changes must be reindexed with the +link:pgm-ScanTrackingIds.html[ScanTrackingIds] program. The tracking ids are serachable using tr: or bug:. diff --git a/Documentation/pgm-ScanTrackingIds.txt b/Documentation/pgm-ScanTrackingIds.txt new file mode 100644 index 0000000000..4ab4a02070 --- /dev/null +++ b/Documentation/pgm-ScanTrackingIds.txt @@ -0,0 +1,51 @@ +ScanTrackingIds +=============== + +NAME +---- +ScanTrackingIds - Rescan changes to index trackingids + +SYNOPSIS +-------- +[verse] +'java' -jar gerrit.war 'ScanTrackingIds' -d + +DESCRIPTION +----------- +Scans every known change and updates the indexed tracking +ids associated with the change, after editing the trackingid +configuration in gerrit.config. + +This task can take quite some time, but can run in the background +concurrently to the server if the database is MySQL or PostgreSQL. +If the database is H2, this task must be run by itself. + +OPTIONS +------- + +-d:: +\--site-path:: + Location of the gerrit.config file, and all other per-site + configuration data, supporting libaries and log files. + +\--threads:: + Number of threads to perform the scan work with. Defaults to + twice the number of CPUs available. + +CONTEXT +------- +This command can only be run on a server which has direct +connectivity to the metadata database, and local access to the +managed Git repositories. + +EXAMPLES +-------- +To rescan all known trackingids: + +==== + $ java -jar gerrit.war ScanTrackingIds -d site_path --threads 16 +==== + +GERRIT +------ +Part of link:index.html[Gerrit Code Review] diff --git a/Documentation/pgm-index.txt b/Documentation/pgm-index.txt index ad54e09826..c6430adc35 100644 --- a/Documentation/pgm-index.txt +++ b/Documentation/pgm-index.txt @@ -18,6 +18,9 @@ link:pgm-daemon.html[daemon]:: link:pgm-gsql.html[gsql]:: Administrative interface to idle database. +link:pgm-ScanTrackingIds.html[ScanTrackingIds]:: + Rescan all changes after configuring trackingids. + version:: Display the release version of Gerrit Code Review. diff --git a/ReleaseNotes/ReleaseNotes-2.1.3.txt b/ReleaseNotes/ReleaseNotes-2.1.3.txt index bd4718b446..a89a2853a9 100644 --- a/ReleaseNotes/ReleaseNotes-2.1.3.txt +++ b/ReleaseNotes/ReleaseNotes-2.1.3.txt @@ -35,7 +35,8 @@ id numbers. Site administrators can configure trackingid sections in gerrit.config to parse and extract issue tracking links from a commit message's footer, and have them indexed by Gerrit. Users can search for relevant changes using the search operator -`tr:`, for example `tr:432181`. +`tr:`, for example `tr:432181`. Administrators can index existing +change records using the ScanTrackingIds program. * List branches/tags containing a merged change + diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/ScanTrackingIds.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/ScanTrackingIds.java new file mode 100644 index 0000000000..e62fe40050 --- /dev/null +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/ScanTrackingIds.java @@ -0,0 +1,182 @@ +// Copyright (C) 2010 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.pgm; + +import static com.google.gerrit.server.schema.DataSourceProvider.Context.MULTI_USER; + +import com.google.gerrit.lifecycle.LifecycleManager; +import com.google.gerrit.lifecycle.LifecycleModule; +import com.google.gerrit.pgm.util.SiteProgram; +import com.google.gerrit.reviewdb.Change; +import com.google.gerrit.reviewdb.PatchSet; +import com.google.gerrit.reviewdb.Project; +import com.google.gerrit.reviewdb.ReviewDb; +import com.google.gerrit.server.ChangeUtil; +import com.google.gerrit.server.config.TrackingFooters; +import com.google.gerrit.server.git.GitRepositoryManager; +import com.google.gerrit.server.git.LocalDiskRepositoryManager; +import com.google.gwtorm.client.OrmException; +import com.google.gwtorm.client.SchemaFactory; +import com.google.inject.Inject; +import com.google.inject.Injector; + +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.RepositoryNotFoundException; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.TextProgressMonitor; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.kohsuke.args4j.Option; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** Scan changes and update the trackingid information for them. */ +public class ScanTrackingIds extends SiteProgram { + @Option(name = "--threads", usage = "Number of concurrent threads to run") + private int threads = 2 * Runtime.getRuntime().availableProcessors(); + + private final LifecycleManager manager = new LifecycleManager(); + private final TextProgressMonitor monitor = new TextProgressMonitor(); + private List todo; + + private Injector dbInjector; + private Injector gitInjector; + + @Inject + private TrackingFooters footers; + + @Inject + private GitRepositoryManager gitManager; + + @Inject + private SchemaFactory database; + + @Override + public int run() throws Exception { + if (threads <= 0) { + threads = 1; + } + + dbInjector = createDbInjector(MULTI_USER); + gitInjector = dbInjector.createChildInjector(new LifecycleModule() { + @Override + protected void configure() { + bind(GitRepositoryManager.class).to(LocalDiskRepositoryManager.class); + listener().to(LocalDiskRepositoryManager.Lifecycle.class); + } + }); + + manager.add(dbInjector, gitInjector); + manager.start(); + gitInjector.injectMembers(this); + + final ReviewDb db = database.open(); + try { + todo = db.changes().all().toList(); + synchronized (monitor) { + monitor.beginTask("Scanning changes", todo.size()); + } + } finally { + db.close(); + } + + final List workers = new ArrayList(threads); + for (int tid = 0; tid < threads; tid++) { + Worker t = new Worker(); + t.start(); + workers.add(t); + } + for (Worker t : workers) { + t.join(); + } + synchronized (monitor) { + monitor.endTask(); + } + manager.stop(); + return 0; + } + + private void scan(ReviewDb db, Change change) { + final Project.NameKey project = change.getDest().getParentKey(); + final Repository git; + try { + git = gitManager.openRepository(project.get()); + } catch (RepositoryNotFoundException e) { + return; + } + try { + PatchSet ps = db.patchSets().get(change.currentPatchSetId()); + if (ps == null || ps.getRevision() == null + || ps.getRevision().get() == null) { + return; + } + ChangeUtil.updateTrackingIds(db, change, footers, parse(git, ps) + .getFooterLines()); + } catch (OrmException error) { + System.err.println("ERR " + error.getMessage()); + } catch (IOException error) { + System.err.println("ERR Cannot scan " + change.getId() + ": " + + error.getMessage()); + } finally { + git.close(); + } + } + + private RevCommit parse(final Repository git, PatchSet ps) + throws MissingObjectException, IncorrectObjectTypeException, IOException { + return new RevWalk(git).parseCommit(ObjectId.fromString(ps.getRevision() + .get())); + } + + private Change next() { + synchronized (todo) { + if (todo.isEmpty()) { + return null; + } + return todo.remove(todo.size() - 1); + } + } + + private class Worker extends Thread { + @Override + public void run() { + ReviewDb db; + try { + db = database.open(); + } catch (OrmException e) { + e.printStackTrace(); + return; + } + try { + for (;;) { + Change change = next(); + if (change == null) { + break; + } + scan(db, change); + synchronized (monitor) { + monitor.update(1); + } + } + } finally { + db.close(); + } + } + } +} diff --git a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/ChangeAccess.java b/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/ChangeAccess.java index b589877f8f..dba2a5824b 100644 --- a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/ChangeAccess.java +++ b/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/ChangeAccess.java @@ -90,4 +90,7 @@ public interface ChangeAccess extends Access { @Query("WHERE open = false AND status = ? AND sortKey < ? ORDER BY sortKey DESC LIMIT ?") ResultSet allClosedNext(char status, String sortKey, int limit) throws OrmException; + + @Query + ResultSet all() throws OrmException; } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java b/gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java index 2f28f984ed..740ce9c527 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java @@ -20,17 +20,24 @@ import com.google.gerrit.reviewdb.Change; import com.google.gerrit.reviewdb.PatchSet; import com.google.gerrit.reviewdb.PatchSetApproval; import com.google.gerrit.reviewdb.ReviewDb; +import com.google.gerrit.reviewdb.TrackingId; +import com.google.gerrit.server.config.TrackingFooter; +import com.google.gerrit.server.config.TrackingFooters; import com.google.gerrit.server.git.MergeQueue; import com.google.gwtorm.client.AtomicUpdate; import com.google.gwtorm.client.OrmConcurrencyException; import com.google.gwtorm.client.OrmException; +import org.eclipse.jgit.revwalk.FooterLine; import org.eclipse.jgit.util.Base64; import org.eclipse.jgit.util.NB; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; public class ChangeUtil { private static int uuidPrefix; @@ -75,6 +82,59 @@ public class ChangeUtil { computeSortKey(c); } + public static void updateTrackingIds(ReviewDb db, Change change, + TrackingFooters trackingFooters, List footerLines) + throws OrmException { + if (trackingFooters.getTrackingFooters().isEmpty() || footerLines.isEmpty()) { + return; + } + + final Set want = new HashSet(); + final Set have = new HashSet( // + db.trackingIds().byChange(change.getId()).toList()); + + for (final TrackingFooter footer : trackingFooters.getTrackingFooters()) { + for (final FooterLine footerLine : footerLines) { + if (footerLine.matches(footer.footerKey())) { + // supporting multiple tracking-ids on a single line + final Matcher m = footer.match().matcher(footerLine.getValue()); + while (m.find()) { + if (m.group().isEmpty()) { + continue; + } + + String idstr; + if (m.groupCount() > 0) { + idstr = m.group(1); + } else { + idstr = m.group(); + } + + if (idstr.isEmpty()) { + continue; + } + if (idstr.length() > TrackingId.TRACKING_ID_MAX_CHAR) { + continue; + } + + want.add(new TrackingId(change.getId(), idstr, footer.system())); + } + } + } + } + + // Only insert the rows we don't have, and delete rows we don't match. + // + final Set toInsert = new HashSet(want); + final Set toDelete = new HashSet(have); + + toInsert.removeAll(have); + toDelete.removeAll(want); + + db.trackingIds().insert(toInsert); + db.trackingIds().delete(toDelete); + } + public static void submit(PatchSet.Id patchSetId, IdentifiedUser user, ReviewDb db, MergeQueue merger) throws OrmException { final Change.Id changeId = patchSetId.getParentKey(); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java index 899f377a20..231d31d789 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java @@ -894,7 +894,7 @@ public class ReceiveCommits implements PreReceiveHook, PostReceiveHook { log.error("Cannot send email for new change " + change.getId(), e); } - addTrackingIds(change, footerLines); + ChangeUtil.updateTrackingIds(db, change, trackingFooters, footerLines); hooks.doPatchsetCreatedHook(change, ps); } @@ -1194,63 +1194,11 @@ public class ReceiveCommits implements PreReceiveHook, PostReceiveHook { log.error("Cannot send email for new patch set " + ps.getId(), e); } - addTrackingIds(change, footerLines); + ChangeUtil.updateTrackingIds(db, change, trackingFooters, footerLines); sendMergedEmail(result); return result != null ? result.info.getKey() : null; } - private void addTrackingIds(final Change change, - final List footerLines) throws OrmException { - if (trackingFooters.getTrackingFooters().isEmpty() || footerLines.isEmpty()) { - return; - } - - final Set want = new HashSet(); - final Set have = new HashSet( // - db.trackingIds().byChange(change.getId()).toList()); - - for (final TrackingFooter footer : trackingFooters.getTrackingFooters()) { - for (final FooterLine footerLine : footerLines) { - if (footerLine.matches(footer.footerKey())) { - // supporting multiple tracking-ids on a single line - final Matcher m = footer.match().matcher(footerLine.getValue()); - while (m.find()) { - if (m.group().isEmpty()) { - continue; - } - - String idstr; - if (m.groupCount() > 0) { - idstr = m.group(1); - } else { - idstr = m.group(); - } - - if (idstr.isEmpty()) { - continue; - } - if (idstr.length() > TrackingId.TRACKING_ID_MAX_CHAR) { - continue; - } - - want.add(new TrackingId(change.getId(), idstr, footer.system())); - } - } - } - } - - // Only insert the rows we don't have, and delete rows we don't match. - // - final Set toInsert = new HashSet(want); - final Set toDelete = new HashSet(have); - - toInsert.removeAll(have); - toDelete.removeAll(want); - - db.trackingIds().insert(toInsert); - db.trackingIds().delete(toDelete); - } - static boolean parentsEqual(RevCommit a, RevCommit b) { if (a.getParentCount() != b.getParentCount()) { return false;