opendev/gerrit
Code Issues Proposed changes

Factor out implicit ref cache from ReceiveCommits and provide NoCache

Browse Source 
ReceiveCommits used to cache the refs that got advertised during 'git
push' in multiple maps directly in the class. This was essentially a
cache, though it was not declared as such. The cache offered fast
lookups for refs-by-change-id and refs-by-object-id (inverse).

This commit cleans up this code by factoring out the caching logic into
a separate class with an interface definition that is reasonably close
to JGit's RefDatabase. Surrounding code was refactored where required.

The intention of this commit is:
1) Clean up existing code
2) Make it easier to re-use the cached refs in other parts of
   ReceiveCommits
3) Allow implementations of RefDatabase that offer fast name-to-ref and
   object-id-to-ref lookups (such as RefTable) to turn off the cache
   using a config. This will be added in a future commit.

Change-Id: I0e5fc9a195df65c13dc2637e157d20c6c8f3afba
This commit is contained in:
Patrick Hiesel
2019-10-17 18:34:25 +02:00
parent ff147dfe2d
commit f97d018b46
9 changed files with 463 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
java/com/google/gerrit/server/BUILD
View File

@@ -54,6 +54,7 @@ java_library(
"//java/com/google/gerrit/prettify:server",
"//java/com/google/gerrit/proto",
"//java/com/google/gerrit/server/cache/serialize",
"//java/com/google/gerrit/server/git/receive:ref_cache",
"//java/com/google/gerrit/server/ioutil",
"//java/com/google/gerrit/server/logging",
"//java/com/google/gerrit/server/util/git",

21
java/com/google/gerrit/server/ChangeUtil.java
View File

@@ -25,7 +25,6 @@ import com.google.inject.Singleton;
import java.io.IOException;
import java.security.SecureRandom;
import java.util.Collection;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.stream.Stream;
@@ -49,22 +48,6 @@ public class ChangeUtil {
return UUID_ENCODING.encode(buf, 0, 4) + '_' + UUID_ENCODING.encode(buf, 4, 4);
}
/**
* Get the next patch set ID from a previously-read map of all refs.
*
* @param allRefs map of full ref name to ref.
* @param id previous patch set ID.
* @return next unused patch set ID for the same change, skipping any IDs whose corresponding ref
* names appear in the {@code allRefs} map.
*/
public static PatchSet.Id nextPatchSetIdFromAllRefsMap(Map<String, Ref> allRefs, PatchSet.Id id) {
PatchSet.Id next = nextPatchSetId(id);
while (allRefs.containsKey(next.toRefName())) {
next = nextPatchSetId(next);
}
return next;
}
/**
* Get the next patch set ID from a previously-read map of refs below the change prefix.
*
@@ -95,9 +78,7 @@ public class ChangeUtil {
/**
* Get the next patch set ID just looking at a single previous patch set ID.
*
* <p>This patch set ID may or may not be available in the database; callers that want a
* previously-unused ID should use {@link #nextPatchSetIdFromAllRefsMap} or {@link
* #nextPatchSetIdFromChangeRefs}.
* <p>This patch set ID may or may not be available in the database.
*
* @param id previous patch set ID.
* @return next patch set ID for the same change, incrementing by 1.

53
java/com/google/gerrit/server/git/GroupCollector.java
View File

@@ -23,16 +23,18 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Multimaps;
import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
import com.google.common.collect.SortedSetMultimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.PatchSet;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.server.PatchSetUtil;
import com.google.gerrit.server.change.RevisionResource;
import com.google.gerrit.server.git.receive.ReceivePackRefCache;
import com.google.gerrit.server.notedb.ChangeNotes;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Deque;
@@ -87,11 +89,11 @@ public class GroupCollector {
return rsrc.getPatchSet().groups();
}
private interface Lookup {
interface Lookup {
List<String> lookup(PatchSet.Id psId);
}
private final ListMultimap<ObjectId, PatchSet.Id> patchSetsBySha;
private final ReceivePackRefCache receivePackRefCache;
private final ListMultimap<ObjectId, String> groups;
private final SetMultimap<String, String> groupAliases;
private final Lookup groupLookup;
@@ -99,12 +101,12 @@ public class GroupCollector {
private boolean done;
public static GroupCollector create(
ListMultimap<ObjectId, Ref> changeRefsById,
ReceivePackRefCache receivePackRefCache,
PatchSetUtil psUtil,
ChangeNotes.Factory notesFactory,
Project.NameKey project) {
return new GroupCollector(
transformRefs(changeRefsById),
receivePackRefCache,
psId -> {
// TODO(dborowitz): Reuse open repository from caller.
ChangeNotes notes = notesFactory.createChecked(project, psId.changeId());
@@ -113,31 +115,15 @@ public class GroupCollector {
});
}
private GroupCollector(ListMultimap<ObjectId, PatchSet.Id> patchSetsBySha, Lookup groupLookup) {
this.patchSetsBySha = patchSetsBySha;
@VisibleForTesting
GroupCollector(ReceivePackRefCache receivePackRefCache, Lookup groupLookup) {
this.receivePackRefCache = receivePackRefCache;
this.groupLookup = groupLookup;
groups = MultimapBuilder.hashKeys().arrayListValues().build();
groupAliases = MultimapBuilder.hashKeys().hashSetValues().build();
}
private static ListMultimap<ObjectId, PatchSet.Id> transformRefs(
ListMultimap<ObjectId, Ref> refs) {
return Multimaps.transformValues(refs, r -> PatchSet.Id.fromRef(r.getName()));
}
@VisibleForTesting
GroupCollector(
ListMultimap<ObjectId, PatchSet.Id> patchSetsBySha,
ListMultimap<PatchSet.Id, String> groupLookup) {
this(
patchSetsBySha,
psId -> {
List<String> groups = groupLookup.get(psId);
return !groups.isEmpty() ? groups : null;
});
}
public void visit(RevCommit c) {
public void visit(RevCommit c) throws IOException {
checkState(!done, "visit() called after getGroups()");
Set<RevCommit> interestingParents = getInterestingParents(c);
@@ -197,7 +183,7 @@ public class GroupCollector {
}
}
public SortedSetMultimap<ObjectId, String> getGroups() {
public SortedSetMultimap<ObjectId, String> getGroups() throws IOException {
done = true;
SortedSetMultimap<ObjectId, String> result =
MultimapBuilder.hashKeys(groups.keySet().size()).treeSetValues().build();
@@ -218,12 +204,13 @@ public class GroupCollector {
return result;
}
private boolean isGroupFromExistingPatchSet(RevCommit commit, String group) {
private boolean isGroupFromExistingPatchSet(RevCommit commit, String group) throws IOException {
ObjectId id = parseGroup(commit, group);
return id != null && patchSetsBySha.containsKey(id);
return id != null && !receivePackRefCache.tipsFromObjectId(id, RefNames.REFS_CHANGES).isEmpty();
}
private Set<String> resolveGroups(ObjectId forCommit, Collection<String> candidates) {
private Set<String> resolveGroups(ObjectId forCommit, Collection<String> candidates)
throws IOException {
Set<String> actual = Sets.newTreeSet();
Set<String> done = Sets.newHashSetWithExpectedSize(candidates.size());
Set<String> seen = Sets.newHashSetWithExpectedSize(candidates.size());
@@ -258,10 +245,13 @@ public class GroupCollector {
}
}
private Iterable<String> resolveGroup(ObjectId forCommit, String group) {
private Iterable<String> resolveGroup(ObjectId forCommit, String group) throws IOException {
ObjectId id = parseGroup(forCommit, group);
if (id != null) {
PatchSet.Id psId = Iterables.getFirst(patchSetsBySha.get(id), null);
Ref ref =
Iterables.getFirst(receivePackRefCache.tipsFromObjectId(id, RefNames.REFS_CHANGES), null);
if (ref != null) {
PatchSet.Id psId = PatchSet.Id.fromRef(ref.getName());
if (psId != null) {
List<String> groups = groupLookup.lookup(psId);
// Group for existing patch set may be missing, e.g. if group has not
@@ -271,6 +261,7 @@ public class GroupCollector {
}
}
}
}
return ImmutableList.of(group);
}
}

17
java/com/google/gerrit/server/git/receive/BUILD
View File

@@ -2,9 +2,13 @@ load("@rules_java//java:defs.bzl", "java_library")
java_library(
name = "receive",
srcs = glob(["**/*.java"]),
srcs = glob(
["**/*.java"],
exclude = ["ReceivePackRefCache.java"],
),
visibility = ["//visibility:public"],
deps = [
":ref_cache",
"//java/com/google/gerrit/common:annotations",
"//java/com/google/gerrit/common:server",
"//java/com/google/gerrit/entities",
@@ -26,3 +30,14 @@ java_library(
"//lib/guice:guice-assistedinject",
],
)
java_library(
name = "ref_cache",
srcs = glob(["ReceivePackRefCache.java"]),
visibility = ["//visibility:public"],
deps = [
"//java/com/google/gerrit/entities",
"//lib:guava",
"//lib:jgit",
],
)

143
java/com/google/gerrit/server/git/receive/ReceiveCommits.java
View File

@@ -343,7 +343,6 @@ class ReceiveCommits {
private final SetPrivateOp.Factory setPrivateOpFactory;
// Assisted injected fields.
private final AllRefsWatcher allRefsWatcher;
private final ProjectState projectState;
private final IdentifiedUser user;
private final ReceivePack receivePack;
@@ -363,12 +362,9 @@ class ReceiveCommits {
private final ListMultimap<String, String> errors;
private final ListMultimap<String, String> pushOptions;
private final ReceivePackRefCache receivePackRefCache;
private final Map<Change.Id, ReplaceRequest> replaceByChange;
// Collections lazily populated during processing.
private ListMultimap<Change.Id, Ref> refsByChange;
private ListMultimap<ObjectId, Ref> refsById;
// Other settings populated during processing.
private MagicBranchInput magicBranch;
private boolean newChangeForAllNotInTarget;
@@ -469,7 +465,6 @@ class ReceiveCommits {
this.setPrivateOpFactory = setPrivateOpFactory;
// Assisted injected fields.
this.allRefsWatcher = allRefsWatcher;
this.projectState = projectState;
this.user = user;
this.receivePack = rp;
@@ -501,6 +496,8 @@ class ReceiveCommits {
this.messageSender = messageSender != null ? messageSender : new ReceivePackMessageSender();
this.resultChangeIds = resultChangeIds;
this.loggingTags = ImmutableMap.of();
receivePackRefCache = ReceivePackRefCache.withAdvertisedRefs(() -> allRefsWatcher.getAllRefs());
}
void init() {
@@ -653,8 +650,15 @@ class ReceiveCommits {
Task replaceProgress = progress.beginSubTask("updated", UNKNOWN);
List<CreateRequest> newChanges = Collections.emptyList();
try {
if (magicBranch != null && magicBranch.cmd.getResult() == NOT_ATTEMPTED) {
try {
newChanges = selectNewAndReplacedChangesFromMagicBranch(newProgress);
} catch (IOException e) {
logger.atSevere().withCause(e).log(
"Failed to select new changes in %s", project.getName());
return;
}
}
// Commit validation has already happened, so any changes without Change-Id are for the
@@ -662,8 +666,11 @@ class ReceiveCommits {
warnAboutMissingChangeId(newChanges);
preparePatchSetsForReplace(newChanges);
insertChangesAndPatchSets(newChanges, replaceProgress);
} finally {
newProgress.end();
replaceProgress.end();
}
queueSuccessMessages(newChanges);
logger.atFine().log(
@@ -1850,11 +1857,12 @@ class ReceiveCommits {
reject(cmd, "cannot use merged with base");
return;
}
RevCommit branchTip = readBranchTip(magicBranch.dest);
if (branchTip == null) {
Ref refTip = receivePackRefCache.exactRef(magicBranch.dest.branch());
if (refTip == null) {
reject(cmd, magicBranch.dest.branch() + " not found");
return;
}
RevCommit branchTip = receivePack.getRevWalk().parseCommit(refTip.getObjectId());
if (!walk.isMergedInto(tip, branchTip)) {
reject(cmd, "not merged into branch");
return;
@@ -1891,8 +1899,9 @@ class ReceiveCommits {
}
}
} else if (newChangeForAllNotInTarget) {
RevCommit branchTip = readBranchTip(magicBranch.dest);
if (branchTip != null) {
Ref refTip = receivePackRefCache.exactRef(magicBranch.dest.branch());
if (refTip != null) {
RevCommit branchTip = receivePack.getRevWalk().parseCommit(refTip.getObjectId());
magicBranch.baseCommit = Collections.singletonList(branchTip);
logger.atFine().log("Set baseCommit = %s", magicBranch.baseCommit.get(0).name());
} else {
@@ -1986,14 +1995,6 @@ class ReceiveCommits {
}
}
private RevCommit readBranchTip(BranchNameKey branch) throws IOException {
Ref r = allRefs().get(branch.branch());
if (r == null) {
return null;
}
return receivePack.getRevWalk().parseCommit(r.getObjectId());
}
/**
* Update an existing change. If draft comments are to be published, these are validated and may
* be withheld.
@@ -2001,7 +2002,8 @@ class ReceiveCommits {
* @return True if the command succeeded, false if it was rejected.
*/
private boolean requestReplaceAndValidateComments(
ReceiveCommand cmd, boolean checkMergedInto, Change change, RevCommit newCommit) {
ReceiveCommand cmd, boolean checkMergedInto, Change change, RevCommit newCommit)
throws IOException {
try (TraceTimer traceTimer = newTimer("requestReplaceAndValidateComments")) {
if (change.isClosed()) {
reject(
@@ -2063,14 +2065,14 @@ class ReceiveCommits {
}
}
private List<CreateRequest> selectNewAndReplacedChangesFromMagicBranch(Task newProgress) {
private List<CreateRequest> selectNewAndReplacedChangesFromMagicBranch(Task newProgress)
throws IOException {
try (TraceTimer traceTimer = newTimer("selectNewAndReplacedChangesFromMagicBranch")) {
logger.atFine().log("Finding new and replaced changes");
List<CreateRequest> newChanges = new ArrayList<>();
ListMultimap<ObjectId, Ref> existing = changeRefsById();
GroupCollector groupCollector =
GroupCollector.create(changeRefsById(), psUtil, notesFactory, project.getNameKey());
GroupCollector.create(receivePackRefCache, psUtil, notesFactory, project.getNameKey());
BranchCommitValidator validator =
commitValidatorFactory.create(projectState, magicBranch.dest, user);
@@ -2111,7 +2113,8 @@ class ReceiveCommits {
receivePack.getRevWalk().parseBody(c);
String name = c.name();
groupCollector.visit(c);
Collection<Ref> existingRefs = existing.get(c);
Collection<Ref> existingRefs =
receivePackRefCache.tipsFromObjectId(c, RefNames.REFS_CHANGES);
if (rejectImplicitMerges) {
Collections.addAll(mergedParents, c.getParents());
@@ -2275,7 +2278,8 @@ class ReceiveCommits {
// In case the change look up from the index failed,
// double check against the existing refs
if (foundInExistingRef(existing.get(p.commit))) {
if (foundInExistingRef(
receivePackRefCache.tipsFromObjectId(p.commit, RefNames.REFS_CHANGES))) {
if (pending.size() == 1) {
reject(magicBranch.cmd, "commit(s) already exists (as current patchset)");
return Collections.emptyList();
@@ -2382,7 +2386,7 @@ class ReceiveCommits {
for (RevCommit c : magicBranch.baseCommit) {
receivePack.getRevWalk().markUninteresting(c);
}
Ref targetRef = allRefs().get(magicBranch.dest.branch());
Ref targetRef = receivePackRefCache.exactRef(magicBranch.dest.branch());
if (targetRef != null) {
logger.atFine().log(
"Marking target ref %s (%s) uninteresting",
@@ -2397,7 +2401,7 @@ class ReceiveCommits {
private void rejectImplicitMerges(Set<RevCommit> mergedParents) throws IOException {
try (TraceTimer traceTimer = newTimer("rejectImplicitMerges")) {
if (!mergedParents.isEmpty()) {
Ref targetRef = allRefs().get(magicBranch.dest.branch());
Ref targetRef = receivePackRefCache.exactRef(magicBranch.dest.branch());
if (targetRef != null) {
RevWalk rw = receivePack.getRevWalk();
RevCommit tip = rw.parseCommit(targetRef.getObjectId());
@@ -2432,13 +2436,15 @@ class ReceiveCommits {
// Mark all branch tips as uninteresting in the given revwalk,
// so we get only the new commits when walking rw.
private void markHeadsAsUninteresting(RevWalk rw, @Nullable String forRef) {
private void markHeadsAsUninteresting(RevWalk rw, @Nullable String forRef) throws IOException {
try (TraceTimer traceTimer =
newTimer("markHeadsAsUninteresting", Metadata.builder().branchName(forRef))) {
int i = 0;
for (Ref ref : allRefs().values()) {
if ((ref.getName().startsWith(R_HEADS) || ref.getName().equals(forRef))
&& ref.getObjectId() != null) {
for (Ref ref :
Iterables.concat(
receivePackRefCache.byPrefix(R_HEADS),
Collections.singletonList(receivePackRefCache.exactRef(forRef)))) {
if (ref != null && ref.getObjectId() != null) {
try {
rw.markUninteresting(rw.parseCommit(ref.getObjectId()));
i++;
@@ -2703,7 +2709,8 @@ class ReceiveCommits {
ReplaceOp replaceOp;
ReplaceRequest(
Change.Id toChange, RevCommit newCommit, ReceiveCommand cmd, boolean checkMergedInto) {
Change.Id toChange, RevCommit newCommit, ReceiveCommand cmd, boolean checkMergedInto)
throws IOException {
this.ontoChange = toChange;
this.newCommitId = newCommit.copy();
this.inputCommand = requireNonNull(cmd);
@@ -2715,11 +2722,12 @@ class ReceiveCommits {
revCommit = null;
}
revisions = HashBiMap.create();
for (Ref ref : refs(toChange)) {
for (Ref ref : receivePackRefCache.byPrefix(RefNames.changeRefPrefix(toChange))) {
try {
revisions.forcePut(
receivePack.getRevWalk().parseCommit(ref.getObjectId()),
PatchSet.Id.fromRef(ref.getName()));
PatchSet.Id psId = PatchSet.Id.fromRef(ref.getName());
if (psId != null) {
revisions.forcePut(receivePack.getRevWalk().parseCommit(ref.getObjectId()), psId);
}
} catch (IOException err) {
logger.atWarning().withCause(err).log(
"Project %s contains invalid change ref %s", project.getName(), ref.getName());
@@ -2950,14 +2958,20 @@ class ReceiveCommits {
private void newPatchSet() throws IOException {
try (TraceTimer traceTimer = newTimer("newPatchSet")) {
RevCommit newCommit = receivePack.getRevWalk().parseCommit(newCommitId);
psId =
ChangeUtil.nextPatchSetIdFromAllRefsMap(
allRefs(), notes.getChange().currentPatchSetId());
psId = nextPatchSetId(notes.getChange().currentPatchSetId());
info = patchSetInfoFactory.get(receivePack.getRevWalk(), newCommit, psId);
cmd = new ReceiveCommand(ObjectId.zeroId(), newCommitId, psId.toRefName());
}
}
private PatchSet.Id nextPatchSetId(PatchSet.Id psId) throws IOException {
PatchSet.Id next = ChangeUtil.nextPatchSetId(psId);
while (receivePackRefCache.exactRef(next.toRefName()) != null) {
next = ChangeUtil.nextPatchSetId(next);
}
return next;
}
void addOps(BatchUpdate bu, @Nullable Task progress) throws IOException {
try (TraceTimer traceTimer = newTimer("addOps")) {
if (magicBranch != null && magicBranch.edit) {
@@ -3091,45 +3105,6 @@ class ReceiveCommits {
}
}
private List<Ref> refs(Change.Id changeId) {
return refsByChange().get(changeId);
}
private void initChangeRefMaps() {
if (refsByChange != null) {
return;
}
try (TraceTimer traceTimer = newTimer("initChangeRefMaps")) {
int estRefsPerChange = 4;
refsById = MultimapBuilder.hashKeys().arrayListValues().build();
refsByChange =
MultimapBuilder.hashKeys(allRefs().size() / estRefsPerChange)
.arrayListValues(estRefsPerChange)
.build();
for (Ref ref : allRefs().values()) {
ObjectId obj = ref.getObjectId();
if (obj != null) {
PatchSet.Id psId = PatchSet.Id.fromRef(ref.getName());
if (psId != null) {
refsById.put(obj, ref);
refsByChange.put(psId.changeId(), ref);
}
}
}
}
}
private ListMultimap<Change.Id, Ref> refsByChange() {
initChangeRefMaps();
return refsByChange;
}
private ListMultimap<ObjectId, Ref> changeRefsById() {
initChangeRefMaps();
return refsById;
}
private static boolean parentsEqual(RevCommit a, RevCommit b) {
if (a.getParentCount() != b.getParentCount()) {
return false;
@@ -3214,7 +3189,6 @@ class ReceiveCommits {
if (!(parsedObject instanceof RevCommit)) {
return;
}
ListMultimap<ObjectId, Ref> existing = changeRefsById();
walk.markStart((RevCommit) parsedObject);
markHeadsAsUninteresting(walk, cmd.getRefName());
int limit = receiveConfig.maxBatchCommits;
@@ -3231,7 +3205,7 @@ class ReceiveCommits {
"more than %d commits, and %s not set", limit, PUSH_OPTION_SKIP_VALIDATION));
return;
}
if (existing.keySet().contains(c)) {
if (!receivePackRefCache.tipsFromObjectId(c, RefNames.REFS_CHANGES).isEmpty()) {
continue;
}
@@ -3281,7 +3255,6 @@ class ReceiveCommits {
rw.markUninteresting(rw.parseCommit(cmd.getOldId()));
}
ListMultimap<ObjectId, Ref> byCommit = changeRefsById();
Map<Change.Key, ChangeNotes> byKey = null;
List<ReplaceRequest> replaceAndClose = new ArrayList<>();
@@ -3291,7 +3264,8 @@ class ReceiveCommits {
for (RevCommit c; (c = rw.next()) != null; ) {
rw.parseBody(c);
for (Ref ref : byCommit.get(c.copy())) {
for (Ref ref :
receivePackRefCache.tipsFromObjectId(c.copy(), RefNames.REFS_CHANGES)) {
PatchSet.Id psId = PatchSet.Id.fromRef(ref.getName());
Optional<ChangeNotes> notes = getChangeNotes(psId.changeId());
if (notes.isPresent() && notes.get().getChange().getDest().equals(branch)) {
@@ -3403,13 +3377,6 @@ class ReceiveCommits {
}
}
// allRefsWatcher hooks into the protocol negotation to get a list of all known refs.
// This is used as a cache of ref -> sha1 values, and to build an inverse index
// of (change => list of refs) and a (SHA1 => refs).
private Map<String, Ref> allRefs() {
return allRefsWatcher.getAllRefs();
}
private TraceTimer newTimer(String name) {
return newTimer(getClass(), name);
}

174
java/com/google/gerrit/server/git/receive/ReceivePackRefCache.java Normal file
View File

@@ -0,0 +1,174 @@
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.git.receive;
import static com.google.common.collect.ImmutableList.toImmutableList;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.MultimapBuilder;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.RefNames;
import java.io.IOException;
import java.util.Map;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefDatabase;
/**
* Simple cache for accessing refs by name, prefix or {@link ObjectId}. Intended to be used when
* processing a {@code git push}.
*
* <p>This class is not thread safe.
*/
public interface ReceivePackRefCache {
/**
* Returns an instance that delegates all calls to the provided {@link RefDatabase}. To be used in
* tests or when the ref database is fast with forward (name to {@link ObjectId}) and inverse
* ({@code ObjectId} to name) lookups.
*/
static ReceivePackRefCache noCache(RefDatabase delegate) {
return new NoCache(delegate);
}
/**
* Returns an instance that answers calls based on refs previously advertised and captured in
* {@link AllRefsWatcher}. Speeds up inverse lookups by building a {@code Map<ObjectId,
* List<Ref>>} and a {@code Map<Change.Id, List<Ref>>}.
*
* <p>This implementation speeds up lookups when the ref database does not support inverse ({@code
* ObjectId} to name) lookups.
*/
static ReceivePackRefCache withAdvertisedRefs(Supplier<Map<String, Ref>> allRefsSupplier) {
return new WithAdvertisedRefs(allRefsSupplier);
}
/** Returns a list of refs whose name starts with {@code prefix} that point to {@code id}. */
ImmutableList<Ref> tipsFromObjectId(ObjectId id, @Nullable String prefix) throws IOException;
/** Returns all refs whose name starts with {@code prefix}. */
ImmutableList<Ref> byPrefix(String prefix) throws IOException;
/** Returns a ref whose name matches {@code ref} or {@code null} if such a ref does not exist. */
@Nullable
Ref exactRef(String ref) throws IOException;
class NoCache implements ReceivePackRefCache {
private final RefDatabase delegate;
private NoCache(RefDatabase delegate) {
this.delegate = delegate;
}
@Override
public ImmutableList<Ref> tipsFromObjectId(ObjectId id, @Nullable String prefix)
throws IOException {
return delegate.getTipsWithSha1(id).stream()
.filter(r -> prefix == null || r.getName().startsWith(prefix))
.collect(toImmutableList());
}
@Override
public ImmutableList<Ref> byPrefix(String prefix) throws IOException {
return delegate.getRefsByPrefix(prefix).stream().collect(toImmutableList());
}
@Override
@Nullable
public Ref exactRef(String name) throws IOException {
return delegate.exactRef(name);
}
}
class WithAdvertisedRefs implements ReceivePackRefCache {
/** We estimate that a change has an average of 4 patch sets plus the meta ref. */
private static final int ESTIMATED_NUMBER_OF_REFS_PER_CHANGE = 5;
private final Supplier<Map<String, Ref>> allRefsSupplier;
// Collections lazily populated during processing.
private Map<String, Ref> allRefs;
/** Contains only patch set refs. */
private ListMultimap<Change.Id, Ref> refsByChange;
/** Contains all refs. */
private ListMultimap<ObjectId, Ref> refsByObjectId;
private WithAdvertisedRefs(Supplier<Map<String, Ref>> allRefsSupplier) {
this.allRefsSupplier = allRefsSupplier;
}
@Override
public ImmutableList<Ref> tipsFromObjectId(ObjectId id, String prefix) {
lazilyInitRefMaps();
return refsByObjectId.get(id).stream()
.filter(r -> prefix == null || r.getName().startsWith(prefix))
.collect(toImmutableList());
}
@Override
public ImmutableList<Ref> byPrefix(String prefix) {
lazilyInitRefMaps();
if (RefNames.isRefsChanges(prefix)) {
Change.Id cId = Change.Id.fromRefPart(prefix);
if (cId != null) {
return refsByChange.get(cId).stream()
.filter(r -> r.getName().startsWith(prefix))
.collect(toImmutableList());
}
}
return allRefs().values().stream()
.filter(r -> r.getName().startsWith(prefix))
.collect(toImmutableList());
}
@Override
@Nullable
public Ref exactRef(String name) {
return allRefs().get(name);
}
private Map<String, Ref> allRefs() {
if (allRefs == null) {
allRefs = allRefsSupplier.get();
}
return allRefs;
}
private void lazilyInitRefMaps() {
if (refsByChange != null) {
return;
}
refsByObjectId = MultimapBuilder.hashKeys().arrayListValues().build();
refsByChange =
MultimapBuilder.hashKeys(allRefs().size() / ESTIMATED_NUMBER_OF_REFS_PER_CHANGE)
.arrayListValues(ESTIMATED_NUMBER_OF_REFS_PER_CHANGE)
.build();
for (Ref ref : allRefs().values()) {
ObjectId objectId = ref.getObjectId();
if (objectId != null) {
refsByObjectId.put(objectId, ref);
Change.Id changeId = Change.Id.fromRef(ref.getName());
if (changeId != null) {
refsByChange.put(changeId, ref);
}
}
}
}
}
}

1
javatests/com/google/gerrit/server/BUILD
View File

@@ -55,6 +55,7 @@ junit_tests(
"//java/com/google/gerrit/server/account/externalids/testing",
"//java/com/google/gerrit/server/cache/serialize",
"//java/com/google/gerrit/server/cache/testing",
"//java/com/google/gerrit/server/git/receive:ref_cache",
"//java/com/google/gerrit/server/ioutil",
"//java/com/google/gerrit/server/logging",
"//java/com/google/gerrit/server/project/testing:project-test-util",

81
javatests/com/google/gerrit/server/git/GroupCollectorTest.java
View File

@@ -21,10 +21,13 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.SortedSetMultimap;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.PatchSet;
import com.google.gerrit.server.git.receive.ReceivePackRefCache;
import java.io.IOException;
import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription;
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevSort;
import org.eclipse.jgit.revwalk.RevWalk;
@@ -44,8 +47,7 @@ public class GroupCollectorTest {
RevCommit branchTip = tr.commit().create();
RevCommit a = tr.commit().parent(branchTip).create();
SortedSetMultimap<ObjectId, String> groups =
collectGroups(newWalk(a, branchTip), patchSets(), groups());
SortedSetMultimap<ObjectId, String> groups = collectGroups(newWalk(a, branchTip), groups());
assertThat(groups).containsEntry(a, a.name());
}
@@ -56,8 +58,7 @@ public class GroupCollectorTest {
RevCommit a = tr.commit().parent(branchTip).create();
RevCommit b = tr.commit().parent(a).create();
SortedSetMultimap<ObjectId, String> groups =
collectGroups(newWalk(b, branchTip), patchSets(), groups());
SortedSetMultimap<ObjectId, String> groups = collectGroups(newWalk(b, branchTip), groups());
assertThat(groups).containsEntry(a, a.name());
assertThat(groups).containsEntry(b, a.name());
@@ -67,12 +68,12 @@ public class GroupCollectorTest {
public void commitWhoseParentIsExistingPatchSetGetsParentsGroup() throws Exception {
RevCommit branchTip = tr.commit().create();
RevCommit a = tr.commit().parent(branchTip).create();
createRef(psId(1, 1), a, tr);
RevCommit b = tr.commit().parent(a).create();
String group = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
SortedSetMultimap<ObjectId, String> groups =
collectGroups(
newWalk(b, branchTip), patchSets().put(a, psId(1, 1)), groups().put(psId(1, 1), group));
collectGroups(newWalk(b, branchTip), groups().put(psId(1, 1), group));
assertThat(groups).containsEntry(a, group);
assertThat(groups).containsEntry(b, group);
@@ -84,8 +85,7 @@ public class GroupCollectorTest {
RevCommit a = tr.commit().parent(branchTip).create();
RevCommit b = tr.commit().parent(a).create();
SortedSetMultimap<ObjectId, String> groups =
collectGroups(newWalk(b, branchTip), patchSets().put(a, psId(1, 1)), groups());
SortedSetMultimap<ObjectId, String> groups = collectGroups(newWalk(b, branchTip), groups());
assertThat(groups).containsEntry(a, a.name());
assertThat(groups).containsEntry(b, a.name());
@@ -98,8 +98,7 @@ public class GroupCollectorTest {
RevCommit b = tr.commit().parent(branchTip).create();
RevCommit m = tr.commit().parent(a).parent(b).create();
SortedSetMultimap<ObjectId, String> groups =
collectGroups(newWalk(m, branchTip), patchSets(), groups());
SortedSetMultimap<ObjectId, String> groups = collectGroups(newWalk(m, branchTip), groups());
assertThat(groups).containsEntry(a, a.name());
assertThat(groups).containsEntry(b, a.name());
@@ -110,13 +109,13 @@ public class GroupCollectorTest {
public void mergeCommitWhereOneParentHasExistingGroup() throws Exception {
RevCommit branchTip = tr.commit().create();
RevCommit a = tr.commit().parent(branchTip).create();
createRef(psId(1, 1), a, tr);
RevCommit b = tr.commit().parent(branchTip).create();
RevCommit m = tr.commit().parent(a).parent(b).create();
String group = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
SortedSetMultimap<ObjectId, String> groups =
collectGroups(
newWalk(m, branchTip), patchSets().put(b, psId(1, 1)), groups().put(psId(1, 1), group));
collectGroups(newWalk(m, branchTip), groups().put(psId(1, 1), group));
// Merge commit and other parent get the existing group.
assertThat(groups).containsEntry(a, group);
@@ -128,16 +127,16 @@ public class GroupCollectorTest {
public void mergeCommitWhereBothParentsHaveDifferentGroups() throws Exception {
RevCommit branchTip = tr.commit().create();
RevCommit a = tr.commit().parent(branchTip).create();
createRef(psId(1, 1), a, tr);
RevCommit b = tr.commit().parent(branchTip).create();
createRef(psId(2, 1), b, tr);
RevCommit m = tr.commit().parent(a).parent(b).create();
String group1 = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
String group2 = "1234567812345678123456781234567812345678";
SortedSetMultimap<ObjectId, String> groups =
collectGroups(
newWalk(m, branchTip),
patchSets().put(a, psId(1, 1)).put(b, psId(2, 1)),
groups().put(psId(1, 1), group1).put(psId(2, 1), group2));
newWalk(m, branchTip), groups().put(psId(1, 1), group1).put(psId(2, 1), group2));
assertThat(groups).containsEntry(a, group1);
assertThat(groups).containsEntry(b, group2);
@@ -149,7 +148,9 @@ public class GroupCollectorTest {
public void mergeCommitMergesGroupsFromParent() throws Exception {
RevCommit branchTip = tr.commit().create();
RevCommit a = tr.commit().parent(branchTip).create();
createRef(psId(1, 1), a, tr);
RevCommit b = tr.commit().parent(branchTip).create();
createRef(psId(2, 1), b, tr);
RevCommit m = tr.commit().parent(a).parent(b).create();
String group1 = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
@@ -158,7 +159,6 @@ public class GroupCollectorTest {
SortedSetMultimap<ObjectId, String> groups =
collectGroups(
newWalk(m, branchTip),
patchSets().put(a, psId(1, 1)).put(b, psId(2, 1)),
groups().put(psId(1, 1), group1).put(psId(2, 1), group2a).put(psId(2, 1), group2b));
assertThat(groups).containsEntry(a, group1);
@@ -171,12 +171,12 @@ public class GroupCollectorTest {
public void mergeCommitWithOneUninterestingParentAndOtherParentIsExisting() throws Exception {
RevCommit branchTip = tr.commit().create();
RevCommit a = tr.commit().parent(branchTip).create();
createRef(psId(1, 1), a, tr);
RevCommit m = tr.commit().parent(branchTip).parent(a).create();
String group = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef";
SortedSetMultimap<ObjectId, String> groups =
collectGroups(
newWalk(m, branchTip), patchSets().put(a, psId(1, 1)), groups().put(psId(1, 1), group));
collectGroups(newWalk(m, branchTip), groups().put(psId(1, 1), group));
assertThat(groups).containsEntry(a, group);
assertThat(groups).containsEntry(m, group);
@@ -188,8 +188,7 @@ public class GroupCollectorTest {
RevCommit a = tr.commit().parent(branchTip).create();
RevCommit m = tr.commit().parent(branchTip).parent(a).create();
SortedSetMultimap<ObjectId, String> groups =
collectGroups(newWalk(m, branchTip), patchSets(), groups());
SortedSetMultimap<ObjectId, String> groups = collectGroups(newWalk(m, branchTip), groups());
assertThat(groups).containsEntry(a, a.name());
assertThat(groups).containsEntry(m, a.name());
@@ -204,8 +203,7 @@ public class GroupCollectorTest {
RevCommit m1 = tr.commit().parent(b).parent(c).create();
RevCommit m2 = tr.commit().parent(a).parent(m1).create();
SortedSetMultimap<ObjectId, String> groups =
collectGroups(newWalk(m2, branchTip), patchSets(), groups());
SortedSetMultimap<ObjectId, String> groups = collectGroups(newWalk(m2, branchTip), groups());
assertThat(groups).containsEntry(a, a.name());
assertThat(groups).containsEntry(b, a.name());
@@ -223,8 +221,7 @@ public class GroupCollectorTest {
assertThat(m.getParentCount()).isEqualTo(2);
assertThat(m.getParent(0)).isEqualTo(m.getParent(1));
SortedSetMultimap<ObjectId, String> groups =
collectGroups(newWalk(m, branchTip), patchSets(), groups());
SortedSetMultimap<ObjectId, String> groups = collectGroups(newWalk(m, branchTip), groups());
assertThat(groups).containsEntry(a, a.name());
assertThat(groups).containsEntry(m, a.name());
@@ -234,7 +231,9 @@ public class GroupCollectorTest {
public void mergeCommitWithOneNewParentAndTwoExistingPatchSets() throws Exception {
RevCommit branchTip = tr.commit().create();
RevCommit a = tr.commit().parent(branchTip).create();
createRef(psId(1, 1), a, tr);
RevCommit b = tr.commit().parent(branchTip).create();
createRef(psId(2, 1), b, tr);
RevCommit c = tr.commit().parent(b).create();
RevCommit m = tr.commit().parent(a).parent(c).create();
@@ -242,9 +241,7 @@ public class GroupCollectorTest {
String group2 = "1234567812345678123456781234567812345678";
SortedSetMultimap<ObjectId, String> groups =
collectGroups(
newWalk(m, branchTip),
patchSets().put(a, psId(1, 1)).put(b, psId(2, 1)),
groups().put(psId(1, 1), group1).put(psId(2, 1), group2));
newWalk(m, branchTip), groups().put(psId(1, 1), group1).put(psId(2, 1), group2));
assertThat(groups).containsEntry(a, group1);
assertThat(groups).containsEntry(b, group2);
@@ -264,16 +261,7 @@ public class GroupCollectorTest {
rw.markStart(rw.parseCommit(d));
// Schema upgrade case: all commits are existing patch sets, but none have
// groups assigned yet.
SortedSetMultimap<ObjectId, String> groups =
collectGroups(
rw,
patchSets()
.put(branchTip, psId(1, 1))
.put(a, psId(2, 1))
.put(b, psId(3, 1))
.put(c, psId(4, 1))
.put(d, psId(5, 1)),
groups());
SortedSetMultimap<ObjectId, String> groups = collectGroups(rw, groups());
assertThat(groups).containsEntry(a, a.name());
assertThat(groups).containsEntry(b, a.name());
@@ -287,6 +275,13 @@ public class GroupCollectorTest {
return PatchSet.id(Change.id(c), p);
}
private static void createRef(PatchSet.Id psId, ObjectId id, TestRepository<?> tr)
throws IOException {
RefUpdate ru = tr.getRepository().updateRef(psId.toRefName());
ru.setNewObjectId(id);
assertThat(ru.update()).isEqualTo(RefUpdate.Result.NEW);
}
private RevWalk newWalk(ObjectId start, ObjectId branchTip) throws Exception {
// Match RevWalk conditions from ReceiveCommits.
RevWalk rw = new RevWalk(tr.getRepository());
@@ -297,12 +292,12 @@ public class GroupCollectorTest {
return rw;
}
private static SortedSetMultimap<ObjectId, String> collectGroups(
RevWalk rw,
ImmutableListMultimap.Builder<ObjectId, PatchSet.Id> patchSetsBySha,
ImmutableListMultimap.Builder<PatchSet.Id, String> groupLookup)
throws Exception {
GroupCollector gc = new GroupCollector(patchSetsBySha.build(), groupLookup.build());
private SortedSetMultimap<ObjectId, String> collectGroups(
RevWalk rw, ImmutableListMultimap.Builder<PatchSet.Id, String> groupLookup) throws Exception {
ImmutableListMultimap<PatchSet.Id, String> groups = groupLookup.build();
GroupCollector gc =
new GroupCollector(
ReceivePackRefCache.noCache(tr.getRepository().getRefDatabase()), (s) -> groups.get(s));
RevCommit c;
while ((c = rw.next()) != null) {
gc.visit(c);

140
javatests/com/google/gerrit/server/git/receive/ReceivePackRefCacheTest.java Normal file
View File

@@ -0,0 +1,140 @@
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.git.receive;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.RefNames;
import java.util.Map;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefDatabase;
import org.junit.Test;
/** Tests for {@link ReceivePackRefCache}. */
public class ReceivePackRefCacheTest {
@Test
public void noCache_prefixDelegatesToRefDb() throws Exception {
Ref ref = newRef("refs/changes/01/1/1", "badc0feebadc0feebadc0feebadc0feebadc0fee");
RefDatabase mockRefDb = mock(RefDatabase.class);
ReceivePackRefCache cache = ReceivePackRefCache.noCache(mockRefDb);
when(mockRefDb.getRefsByPrefix(RefNames.REFS_HEADS)).thenReturn(ImmutableList.of(ref));
assertThat(cache.byPrefix(RefNames.REFS_HEADS)).containsExactly(ref);
verify(mockRefDb).getRefsByPrefix(RefNames.REFS_HEADS);
verifyNoMoreInteractions(mockRefDb);
}
@Test
public void noCache_exactRefDelegatesToRefDb() throws Exception {
Ref ref = newRef("refs/changes/01/1/1", "badc0feebadc0feebadc0feebadc0feebadc0fee");
RefDatabase mockRefDb = mock(RefDatabase.class);
ReceivePackRefCache cache = ReceivePackRefCache.noCache(mockRefDb);
when(mockRefDb.exactRef("refs/heads/master")).thenReturn(ref);
assertThat(cache.exactRef("refs/heads/master")).isEqualTo(ref);
verify(mockRefDb).exactRef("refs/heads/master");
verifyNoMoreInteractions(mockRefDb);
}
@Test
public void noCache_tipsFromObjectIdDelegatesToRefDbAndFiltersByPrefix() throws Exception {
Ref refBla = newRef("refs/bla", "badc0feebadc0feebadc0feebadc0feebadc0fee");
Ref refheads = newRef(RefNames.REFS_HEADS, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef");
RefDatabase mockRefDb = mock(RefDatabase.class);
ReceivePackRefCache cache = ReceivePackRefCache.noCache(mockRefDb);
when(mockRefDb.getTipsWithSha1(ObjectId.zeroId()))
.thenReturn(ImmutableSet.of(refBla, refheads));
assertThat(cache.tipsFromObjectId(ObjectId.zeroId(), RefNames.REFS_HEADS))
.containsExactly(refheads);
verify(mockRefDb).getTipsWithSha1(ObjectId.zeroId());
verifyNoMoreInteractions(mockRefDb);
}
@Test
public void advertisedRefs_prefixScans() throws Exception {
Ref refBla =
new ObjectIdRef.Unpeeled(
Ref.Storage.NEW,
"refs/bla/1",
ObjectId.fromString("badc0feebadc0feebadc0feebadc0feebadc0fee"),
1);
ReceivePackRefCache cache =
ReceivePackRefCache.withAdvertisedRefs(() -> ImmutableMap.of(refBla.getName(), refBla));
assertThat(cache.byPrefix("refs/bla")).containsExactly(refBla);
}
@Test
public void advertisedRefs_prefixScansChangeId() throws Exception {
Map<String, Ref> refs = setupTwoChanges();
ReceivePackRefCache cache = ReceivePackRefCache.withAdvertisedRefs(() -> refs);
assertThat(cache.byPrefix(RefNames.changeRefPrefix(Change.id(1))))
.containsExactly(refs.get("refs/changes/01/1/1"));
}
@Test
public void advertisedRefs_exactRef() throws Exception {
Map<String, Ref> refs = setupTwoChanges();
ReceivePackRefCache cache = ReceivePackRefCache.withAdvertisedRefs(() -> refs);
assertThat(cache.exactRef("refs/changes/01/1/1")).isEqualTo(refs.get("refs/changes/01/1/1"));
}
@Test
public void advertisedRefs_tipsFromObjectIdWithNoPrefix() throws Exception {
Map<String, Ref> refs = setupTwoChanges();
ReceivePackRefCache cache = ReceivePackRefCache.withAdvertisedRefs(() -> refs);
assertThat(
cache.tipsFromObjectId(
ObjectId.fromString("badc0feebadc0feebadc0feebadc0feebadc0fee"), null))
.containsExactly(refs.get("refs/changes/01/1/1"));
}
@Test
public void advertisedRefs_tipsFromObjectIdWithPrefix() throws Exception {
Map<String, Ref> refs = setupTwoChanges();
ReceivePackRefCache cache = ReceivePackRefCache.withAdvertisedRefs(() -> refs);
assertThat(
cache.tipsFromObjectId(
ObjectId.fromString("badc0feebadc0feebadc0feebadc0feebadc0fee"), "/refs/some"))
.isEmpty();
}
private static Ref newRef(String name, String sha1) {
return new ObjectIdRef.Unpeeled(Ref.Storage.NEW, name, ObjectId.fromString(sha1), 1);
}
private Map<String, Ref> setupTwoChanges() {
Ref ref1 = newRef("refs/changes/01/1/1", "badc0feebadc0feebadc0feebadc0feebadc0fee");
Ref ref2 = newRef("refs/changes/02/2/1", "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef");
return ImmutableMap.of(ref1.getName(), ref1, ref2.getName(), ref2);
}
}