opendev/gerrit
Code Issues Proposed changes

Add a "groups" field to PatchSet

Browse Source 
This field is intended to be used as an explicit replacement for the
current "Related Changes" heuristic, which may involve walking
arbitrary amounts of history and making lots of database lookups in
the case of some merge topologies. With this field in the index, we
can implement GetRelated using a single secondary index lookup once we
know the patch set's group identifiers. Combined with the stored
PatchSet field, this gives us the entire set of commits that need to
be considered for rendering the Related Changes tab, and the only
remaining work is to do a simple walk to determine the relative topo
ordering.

Any idea of automatically grouping changes together requires some
amount of heuristics, so we use opaque string values in this field in
case we decide to change heuristics later. If we change heuristics,
old groups will still be valid as long as the relevant changes stay
the same. (We might also decide to add a way to manually group changes
together; today, this use case is largely handled by other features
such as topics, but this implementation does not rule that out.)

The field is actually multi-valued, as the heuristics we implemented
in GroupCollector multiple groups if it is the merge commit of two
distinct branches of open changes. However, since we don't need to
look up changes in the database by group, it's not worth the effort
of an extra SQL table, so we just concatenate groups together.

Another useful feature of this implementation is that group
information is preserved when changes are closed, so submitting a
change will no longer immediately cause the Related Changes tab to
disappear.

Use GroupCollector to assign groups in both the main ReceiveCommits
case and the schema upgrade (for all open changes). See the
documentation of GroupCollector for how this is intended to work.

For code paths other than ReceiveCommits, the rules are much simpler,
because there is only one change involved in the operation. Depending
on the context, we simply either copy the existing group of the latest
patch set, or we create a new group.

Change-Id: I7cef275772882b045be14fd9bbbe7c52c73da2a8
This commit is contained in:
Dave Borowitz
2015-05-11 16:02:18 -07:00
parent c6853e158b
commit 336d21313e
10 changed files with 356 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
gerrit-server/src/main/java/com/google/gerrit/server/change/ChangeInserter.java
View File

@@ -33,6 +33,7 @@ import com.google.gerrit.server.ChangeMessagesUtil;
import com.google.gerrit.server.ChangeUtil;
import com.google.gerrit.server.account.AccountCache;
import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
import com.google.gerrit.server.git.GroupCollector;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.index.ChangeIndexer;
import com.google.gerrit.server.mail.CreateChangeSender;
@@ -163,6 +164,11 @@ public class ChangeInserter {
return this;
}
public ChangeInserter setGroups(Iterable<String> groups) {
patchSet.setGroups(groups);
return this;
}
public ChangeInserter setHashtags(Set<String> hashtags) {
this.hashtags = hashtags;
return this;
@@ -205,6 +211,9 @@ public class ChangeInserter {
db.changes().beginTransaction(change.getId());
try {
ChangeUtil.insertAncestors(db, patchSet.getId(), commit);
if (patchSet.getGroups() == null) {
patchSet.setGroups(GroupCollector.getDefaultGroups(patchSet));
}
db.patchSets().insert(Collections.singleton(patchSet));
db.changes().insert(Collections.singleton(change));
LabelTypes labelTypes = projectControl.getLabelTypes();

4
gerrit-server/src/main/java/com/google/gerrit/server/change/CreateChange.java
View File

@@ -157,6 +157,7 @@ public class CreateChange implements
try (Repository git = gitManager.openRepository(project);
RevWalk rw = new RevWalk(git)) {
ObjectId parentCommit;
List<String> groups;
if (input.baseChange != null) {
List<Change> changes = changeUtil.findChanges(input.baseChange);
if (changes.size() != 1) {
@@ -172,6 +173,7 @@ public class CreateChange implements
new PatchSet.Id(change.getId(),
change.currentPatchSetId().get()));
parentCommit = ObjectId.fromString(ps.getRevision().get());
groups = ps.getGroups();
} else {
Ref destRef = git.getRef(refName);
if (destRef == null) {
@@ -179,6 +181,7 @@ public class CreateChange implements
"Branch %s does not exist.", refName));
}
parentCommit = destRef.getObjectId();
groups = null;
}
RevCommit mergeTip = rw.parseCommit(parentCommit);
@@ -208,6 +211,7 @@ public class CreateChange implements
change.setTopic(input.topic);
ins.setDraft(input.status != null && input.status == ChangeStatus.DRAFT);
ins.setGroups(groups);
ins.insert();
return Response.created(json.format(change.getId()));

15
gerrit-server/src/main/java/com/google/gerrit/server/change/PatchSetInserter.java
View File

@@ -35,6 +35,7 @@ import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.events.CommitReceivedEvent;
import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
import com.google.gerrit.server.git.BanCommit;
import com.google.gerrit.server.git.GroupCollector;
import com.google.gerrit.server.git.validators.CommitValidationException;
import com.google.gerrit.server.git.validators.CommitValidators;
import com.google.gerrit.server.index.ChangeIndexer;
@@ -108,6 +109,7 @@ public class PatchSetInserter {
private SshInfo sshInfo;
private ValidatePolicy validatePolicy = ValidatePolicy.GERRIT;
private boolean draft;
private Iterable<String> groups;
private boolean runHooks;
private boolean sendMail;
private Account.Id uploader;
@@ -200,6 +202,11 @@ public class PatchSetInserter {
return this;
}
public PatchSetInserter setGroups(Iterable<String> groups) {
this.groups = groups;
return this;
}
public PatchSetInserter setRunHooks(boolean runHooks) {
this.runHooks = runHooks;
return this;
@@ -239,12 +246,18 @@ public class PatchSetInserter {
db.changes().beginTransaction(c.getId());
try {
if (!db.changes().get(c.getId()).getStatus().isOpen()) {
updatedChange = db.changes().get(c.getId());
if (!updatedChange.getStatus().isOpen()) {
throw new InvalidChangeOperationException(String.format(
"Change %s is closed", c.getId()));
}
ChangeUtil.insertAncestors(db, patchSet.getId(), commit);
if (groups != null) {
patchSet.setGroups(groups);
} else {
patchSet.setGroups(GroupCollector.getCurrentGroups(db, c));
}
db.patchSets().insert(Collections.singleton(patchSet));
SetMultimap<ReviewerState, Account.Id> oldReviewers = sendMail

50
gerrit-server/src/main/java/com/google/gerrit/server/git/GroupCollector.java
View File

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkState;
import static org.eclipse.jgit.revwalk.RevFlag.UNINTERESTING;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
@@ -26,12 +27,17 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Multimaps;
import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.change.RevisionResource;
import com.google.gwtorm.server.OrmException;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,6 +82,25 @@ public class GroupCollector {
private static final Logger log =
LoggerFactory.getLogger(GroupCollector.class);
public static List<String> getCurrentGroups(ReviewDb db, Change c)
throws OrmException {
PatchSet ps = db.patchSets().get(c.currentPatchSetId());
return ps != null ? ps.getGroups() : null;
}
public static List<String> getDefaultGroups(PatchSet ps) {
return ImmutableList.of(ps.getRevision().get());
}
public static List<String> getGroups(RevisionResource rsrc) {
if (rsrc.getEdit().isPresent()) {
// Groups for an edit are just the base revision's groups, since they have
// the same parent.
return rsrc.getEdit().get().getBasePatchSet().getGroups();
}
return rsrc.getPatchSet().getGroups();
}
private static interface Lookup {
List<String> lookup(PatchSet.Id psId) throws OrmException;
}
@@ -96,6 +121,27 @@ public class GroupCollector {
groupAliases = HashMultimap.create();
}
public GroupCollector(
Multimap<ObjectId, Ref> changeRefsById,
final ReviewDb db) {
this(
Multimaps.transformValues(
changeRefsById,
new Function<Ref, PatchSet.Id>() {
@Override
public PatchSet.Id apply(Ref in) {
return PatchSet.Id.fromRef(in.getName());
}
}),
new Lookup() {
@Override
public List<String> lookup(PatchSet.Id psId) throws OrmException {
PatchSet ps = db.patchSets().get(psId);
return ps != null ? ps.getGroups() : null;
}
});
}
@VisibleForTesting
GroupCollector(
Multimap<ObjectId, PatchSet.Id> patchSetsBySha,
@@ -111,7 +157,7 @@ public class GroupCollector {
});
}
void visit(RevCommit c) {
public void visit(RevCommit c) {
checkState(!done, "visit() called after getGroups()");
Set<RevCommit> interestingParents = getInterestingParents(c);
@@ -171,7 +217,7 @@ public class GroupCollector {
}
}
SetMultimap<ObjectId, String> getGroups() throws OrmException {
public SetMultimap<ObjectId, String> getGroups() throws OrmException {
done = true;
SetMultimap<ObjectId, String> result = MultimapBuilder
.hashKeys(groups.keySet().size())

40
gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java
View File

@@ -44,6 +44,7 @@ import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Ordering;
import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
@@ -1467,6 +1468,10 @@ public class ReceiveCommits {
private void selectNewAndReplacedChangesFromMagicBranch() {
newChanges = Lists.newArrayList();
final RevWalk walk = rp.getRevWalk();
Set<ObjectId> existing = changeRefsById().keySet();
GroupCollector groupCollector = new GroupCollector(refsById, db);
walk.reset();
walk.sort(RevSort.TOPO);
walk.sort(RevSort.REVERSE, true);
@@ -1486,7 +1491,6 @@ public class ReceiveCommits {
magicBranch.ctl != null ? magicBranch.ctl.getRefName() : null);
}
Set<ObjectId> existing = changeRefsById().keySet();
List<ChangeLookup> pending = Lists.newArrayList();
final Set<Change.Key> newChangeIds = new HashSet<>();
final int maxBatchChanges =
@@ -1496,7 +1500,17 @@ public class ReceiveCommits {
if (c == null) {
break;
}
groupCollector.visit(c);
if (existing.contains(c)) { // Commit is already tracked.
// TODO(dborowitz): Corner case where an existing commit might need a
// new group:
// Let A<-B<-C, then:
// 1. Push A to refs/heads/master
// 2. Push B to refs/for/master
// 3. Force push A~ to refs/heads/master
// 4. Push C to refs/for/master.
// B will be in existing so we aren't replacing the patch set. It used
// to have its own group, but now needs to to be changed to A's group.
continue;
}
@@ -1605,8 +1619,20 @@ public class ReceiveCommits {
reject(magicBranch.cmd, "edit is not supported for new changes");
return;
}
for (CreateRequest create : newChanges) {
batch.addCommand(create.cmd);
try {
Multimap<ObjectId, String> groups = groupCollector.getGroups();
for (CreateRequest create : newChanges) {
batch.addCommand(create.cmd);
create.groups = groups.get(create.commit);
}
for (ReplaceRequest replace : replaceByChange.values()) {
replace.groups = groups.get(replace.newCommit);
}
} catch (OrmException e) {
log.error("Error collecting groups for changes", e);
reject(magicBranch.cmd, "internal server error");
return;
}
}
@@ -1646,6 +1672,7 @@ public class ReceiveCommits {
final ReceiveCommand cmd;
final ChangeInserter ins;
boolean created;
Collection<String> groups;
CreateRequest(RefControl ctl, RevCommit c, Change.Key changeKey)
throws OrmException {
@@ -1690,7 +1717,7 @@ public class ReceiveCommits {
}
private void insertChange(ReviewDb db) throws OrmException, IOException {
final PatchSet ps = ins.getPatchSet();
final PatchSet ps = ins.setGroups(groups).getPatchSet();
final Account.Id me = currentUser.getAccountId();
final List<FooterLine> footerLines = commit.getFooterLines();
final MailRecipients recipients = new MailRecipients();
@@ -1845,6 +1872,7 @@ public class ReceiveCommits {
String mergedIntoRef;
boolean skip;
private PatchSet.Id priorPatchSet;
Collection<String> groups;
ReplaceRequest(final Change.Id toChange, final RevCommit newCommit,
final ReceiveCommand cmd, final boolean checkMergedInto) {
@@ -2020,6 +2048,7 @@ public class ReceiveCommits {
newPatchSet.setCreatedOn(TimeUtil.nowTs());
newPatchSet.setUploader(currentUser.getAccountId());
newPatchSet.setRevision(toRevId(newCommit));
newPatchSet.setGroups(groups);
if (magicBranch != null && magicBranch.draft) {
newPatchSet.setDraft(true);
}
@@ -2124,6 +2153,9 @@ public class ReceiveCommits {
}
ChangeUtil.insertAncestors(db, newPatchSet.getId(), newCommit);
if (newPatchSet.getGroups() == null) {
newPatchSet.setGroups(GroupCollector.getCurrentGroups(db, change));
}
db.patchSets().insert(Collections.singleton(newPatchSet));
if (checkMergedInto) {

2
gerrit-server/src/main/java/com/google/gerrit/server/git/strategy/CherryPick.java
View File

@@ -27,6 +27,7 @@ import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
import com.google.gerrit.server.git.CodeReviewCommit;
import com.google.gerrit.server.git.CommitMergeStatus;
import com.google.gerrit.server.git.GroupCollector;
import com.google.gerrit.server.git.MergeConflictException;
import com.google.gerrit.server.git.MergeException;
import com.google.gerrit.server.git.MergeIdenticalTreeException;
@@ -186,6 +187,7 @@ public class CherryPick extends SubmitStrategy {
args.db.changes().beginTransaction(n.change().getId());
try {
insertAncestors(args.db, ps.getId(), newCommit);
ps.setGroups(GroupCollector.getCurrentGroups(args.db, n.change()));
args.db.patchSets().insert(Collections.singleton(ps));
n.change()
.setCurrentPatchSet(patchSetInfoFactory.get(newCommit, ps.getId()));

2
gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaVersion.java
View File

@@ -32,7 +32,7 @@ import java.util.List;
/** A version of the database schema. */
public abstract class SchemaVersion {
/** The current schema version. */
public static final Class<Schema_107> C = Schema_107.class;
public static final Class<Schema_108> C = Schema_108.class;
public static int getBinaryVersion() {
return guessVersion(C);

161
gerrit-server/src/main/java/com/google/gerrit/server/schema/Schema_108.java Normal file
View File

@@ -0,0 +1,161 @@
// Copyright (C) 2015 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.schema;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.SetMultimap;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.GroupCollector;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Provider;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevSort;
import org.eclipse.jgit.revwalk.RevWalk;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
public class Schema_108 extends SchemaVersion {
private final GitRepositoryManager repoManager;
@Inject
Schema_108(Provider<Schema_107> prior,
GitRepositoryManager repoManager) {
super(prior);
this.repoManager = repoManager;
}
@Override
protected void migrateData(ReviewDb db, UpdateUI ui) throws OrmException {
ui.message("Listing all changes ...");
SetMultimap<Project.NameKey, Change.Id> openByProject =
getOpenChangesByProject(db);
ui.message("done");
ui.message("Updating groups for open changes ...");
int i = 0;
for (Map.Entry<Project.NameKey, Collection<Change.Id>> e
: openByProject.asMap().entrySet()) {
try (Repository repo = repoManager.openRepository(e.getKey());
RevWalk rw = new RevWalk(repo)) {
updateProjectGroups(db, repo, rw, (Set<Change.Id>) e.getValue());
} catch (IOException err) {
throw new OrmException(err);
}
if (++i % 100 == 0) {
ui.message(" done " + i + " projects ...");
}
}
ui.message("done");
}
private static void updateProjectGroups(ReviewDb db, Repository repo,
RevWalk rw, Set<Change.Id> changes) throws OrmException, IOException {
// Match sorting in ReceiveCommits.
rw.reset();
rw.sort(RevSort.TOPO);
rw.sort(RevSort.REVERSE, true);
RefDatabase refdb = repo.getRefDatabase();
for (Ref ref : refdb.getRefs(Constants.R_HEADS).values()) {
RevCommit c = maybeParseCommit(rw, ref.getObjectId());
if (c != null) {
rw.markUninteresting(c);
}
}
Multimap<ObjectId, Ref> changeRefsBySha = ArrayListMultimap.create();
Multimap<ObjectId, PatchSet.Id> patchSetsBySha = ArrayListMultimap.create();
for (Ref ref : refdb.getRefs(RefNames.REFS_CHANGES).values()) {
ObjectId id = ref.getObjectId();
if (ref.getObjectId() == null) {
continue;
}
id = id.copy();
changeRefsBySha.put(id, ref);
PatchSet.Id psId = PatchSet.Id.fromRef(ref.getName());
if (psId != null && changes.contains(psId.getParentKey())) {
patchSetsBySha.put(id, psId);
RevCommit c = maybeParseCommit(rw, id);
if (c != null) {
rw.markStart(c);
}
}
}
GroupCollector collector = new GroupCollector(changeRefsBySha, db);
RevCommit c;
while ((c = rw.next()) != null) {
collector.visit(c);
}
updateGroups(db, collector, patchSetsBySha);
}
private static void updateGroups(ReviewDb db, GroupCollector collector,
Multimap<ObjectId, PatchSet.Id> patchSetsBySha) throws OrmException {
Map<PatchSet.Id, PatchSet> patchSets =
db.patchSets().toMap(db.patchSets().get(patchSetsBySha.values()));
for (Map.Entry<ObjectId, Collection<String>> e
: collector.getGroups().asMap().entrySet()) {
for (PatchSet.Id psId : patchSetsBySha.get(e.getKey())) {
PatchSet ps = patchSets.get(psId);
if (ps != null) {
ps.setGroups(e.getValue());
}
}
}
db.patchSets().update(patchSets.values());
}
private SetMultimap<Project.NameKey, Change.Id> getOpenChangesByProject(
ReviewDb db) throws OrmException {
SetMultimap<Project.NameKey, Change.Id> openByProject =
HashMultimap.create();
for (Change c : db.changes().all()) {
if (c.getStatus().isOpen()) {
openByProject.put(c.getProject(), c.getId());
}
}
return openByProject;
}
private static RevCommit maybeParseCommit(RevWalk rw, ObjectId id)
throws IOException {
if (id == null) {
return null;
}
RevObject obj = rw.parseAny(id);
return (obj instanceof RevCommit) ? (RevCommit) obj : null;
}
}