opendev/gerrit
Code Issues Proposed changes

Remove remaining NoteDb rebuilding machinery

Browse Source 
Now that we no longer auto-rebuild changes and we have no ReviewDb to
NoteDb migration code, the rebuild package and associated code is
completely unused.

Change-Id: Id4d65ce2bcd6ba828cc39addfe1467fae738eb6b
This commit is contained in:
Edwin Kempin
2018-12-10 17:03:35 +01:00
parent 47d3107031
commit e34923d720
33 changed files with 10 additions and 5882 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
java/com/google/gerrit/acceptance/InMemoryTestingDatabaseModule.java
View File

@@ -28,8 +28,6 @@ import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.config.TrackingFooters;
import com.google.gerrit.server.config.TrackingFootersProvider;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.notedb.ChangeBundleReader;
import com.google.gerrit.server.notedb.GwtormChangeBundleReader;
import com.google.gerrit.server.notedb.NotesMigration;
import com.google.gerrit.server.schema.NotesMigrationSchemaFactory;
import com.google.gerrit.server.schema.ReviewDbFactory;
@@ -83,7 +81,6 @@ class InMemoryTestingDatabaseModule extends LifecycleModule {
bind(schemaFactory).to(NotesMigrationSchemaFactory.class);
bind(Key.get(schemaFactory, ReviewDbFactory.class)).to(InMemoryDatabase.class);
bind(InMemoryDatabase.class).in(SINGLETON);
bind(ChangeBundleReader.class).to(GwtormChangeBundleReader.class);
listener().to(CreateDatabase.class);

8
java/com/google/gerrit/pgm/util/BatchProgramModule.java
View File

@@ -50,7 +50,6 @@ import com.google.gerrit.server.config.CanonicalWebUrlProvider;
import com.google.gerrit.server.config.DefaultUrlFormatter;
import com.google.gerrit.server.config.DisableReverseDnsLookup;
import com.google.gerrit.server.config.DisableReverseDnsLookupProvider;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.GitReceivePackGroups;
import com.google.gerrit.server.config.GitUploadPackGroups;
import com.google.gerrit.server.config.SysExecutorModule;
@@ -87,16 +86,13 @@ import com.google.inject.util.Providers;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.eclipse.jgit.lib.Config;
/** Module for programs that perform batch operations on a site. */
public class BatchProgramModule extends FactoryModule {
private final Config cfg;
private final Module reviewDbModule;
@Inject
BatchProgramModule(@GerritServerConfig Config cfg, PerThreadReviewDbModule reviewDbModule) {
this.cfg = cfg;
BatchProgramModule(PerThreadReviewDbModule reviewDbModule) {
this.reviewDbModule = reviewDbModule;
}
@@ -168,7 +164,7 @@ public class BatchProgramModule extends FactoryModule {
install(new H2CacheModule());
install(new ExternalIdModule());
install(new GroupModule());
install(new NoteDbModule(cfg));
install(new NoteDbModule());
install(AccountCacheImpl.module());
install(GroupCacheImpl.module());
install(GroupIncludeCacheImpl.module());

2
java/com/google/gerrit/server/config/GerritGlobalModule.java
View File

@@ -246,7 +246,7 @@ public class GerritGlobalModule extends FactoryModule {
install(new GitModule());
install(new GroupDbModule());
install(new GroupModule());
install(new NoteDbModule(cfg));
install(new NoteDbModule());
install(new PrologModule());
install(new DefaultSubmitRule.Module());
install(new IgnoreSelfApprovalRule.Module());

976
java/com/google/gerrit/server/notedb/ChangeBundle.java
View File

@@ -1,976 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap;
import static com.google.gerrit.reviewdb.server.ReviewDbUtil.checkColumns;
import static com.google.gerrit.reviewdb.server.ReviewDbUtil.intKeyOrdering;
import static com.google.gerrit.server.notedb.ChangeBundle.Source.NOTE_DB;
import static com.google.gerrit.server.notedb.ChangeBundle.Source.REVIEW_DB;
import static com.google.gerrit.server.util.time.TimeUtil.truncateToSecond;
import static java.util.Comparator.comparing;
import static java.util.Comparator.naturalOrder;
import static java.util.Comparator.nullsFirst;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
import com.google.auto.value.AutoValue;
import com.google.common.base.CharMatcher;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Strings;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.google.common.collect.Streams;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.ChangeMessage;
import com.google.gerrit.reviewdb.client.PatchLineComment;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.PatchSet.Id;
import com.google.gerrit.reviewdb.client.PatchSetApproval;
import com.google.gerrit.server.ChangeUtil;
import com.google.gerrit.server.CommentsUtil;
import com.google.gerrit.server.ReviewerSet;
import com.google.gerrit.server.notedb.rebuild.ChangeRebuilderImpl;
import com.google.gwtorm.client.Column;
import com.google.gwtorm.server.OrmException;
import java.lang.reflect.Field;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
* A bundle of all entities rooted at a single {@link Change} entity.
*
* <p>See the {@link Change} Javadoc for a depiction of this tree. Bundles may be compared using
* {@link #differencesFrom(ChangeBundle)}, which normalizes out the minor implementation differences
* between ReviewDb and NoteDb.
*/
public class ChangeBundle {
public enum Source {
REVIEW_DB,
NOTE_DB;
}
public static ChangeBundle fromNotes(CommentsUtil commentsUtil, ChangeNotes notes)
throws OrmException {
return new ChangeBundle(
notes.getChange(),
notes.getChangeMessages(),
notes.getPatchSets().values(),
notes.getApprovals().values(),
Iterables.concat(
CommentsUtil.toPatchLineComments(
notes.getChangeId(),
PatchLineComment.Status.DRAFT,
commentsUtil.draftByChange(null, notes)),
CommentsUtil.toPatchLineComments(
notes.getChangeId(),
PatchLineComment.Status.PUBLISHED,
commentsUtil.publishedByChange(null, notes))),
notes.getReviewers(),
Source.NOTE_DB);
}
private static ImmutableSortedMap<ChangeMessage.Key, ChangeMessage> changeMessageMap(
Collection<ChangeMessage> in) {
return in.stream()
.collect(
toImmutableSortedMap(
comparing((ChangeMessage.Key k) -> k.getParentKey().get())
.thenComparing(k -> k.get()),
cm -> cm.getKey(),
cm -> cm));
}
// Unlike the *Map comparators, which are intended to make key lists diffable,
// this comparator sorts first on timestamp, then on every other field.
private static final Comparator<ChangeMessage> CHANGE_MESSAGE_COMPARATOR =
comparing(ChangeMessage::getWrittenOn)
.thenComparing(m -> m.getKey().getParentKey().get())
.thenComparing(
m -> m.getPatchSetId() != null ? m.getPatchSetId().get() : null,
nullsFirst(naturalOrder()))
.thenComparing(ChangeMessage::getAuthor, intKeyOrdering())
.thenComparing(ChangeMessage::getMessage, nullsFirst(naturalOrder()));
private static ImmutableList<ChangeMessage> changeMessageList(Iterable<ChangeMessage> in) {
return Streams.stream(in).sorted(CHANGE_MESSAGE_COMPARATOR).collect(toImmutableList());
}
private static ImmutableSortedMap<Id, PatchSet> patchSetMap(Iterable<PatchSet> in) {
return Streams.stream(in)
.collect(toImmutableSortedMap(patchSetIdComparator(), PatchSet::getId, ps -> ps));
}
private static ImmutableSortedMap<PatchSetApproval.Key, PatchSetApproval> patchSetApprovalMap(
Iterable<PatchSetApproval> in) {
return Streams.stream(in)
.collect(
toImmutableSortedMap(
comparing(PatchSetApproval.Key::getParentKey, patchSetIdComparator())
.thenComparing(PatchSetApproval.Key::getAccountId, intKeyOrdering())
.thenComparing(PatchSetApproval.Key::getLabelId),
PatchSetApproval::getKey,
a -> a));
}
private static ImmutableSortedMap<PatchLineComment.Key, PatchLineComment> patchLineCommentMap(
Iterable<PatchLineComment> in) {
return Streams.stream(in)
.collect(
toImmutableSortedMap(
comparing(
(PatchLineComment.Key k) -> k.getParentKey().getParentKey(),
patchSetIdComparator())
.thenComparing(PatchLineComment.Key::getParentKey)
.thenComparing(PatchLineComment.Key::get),
PatchLineComment::getKey,
c -> c));
}
private static Comparator<PatchSet.Id> patchSetIdComparator() {
return comparing((PatchSet.Id id) -> id.getParentKey().get()).thenComparing(id -> id.get());
}
static {
// Initialization-time checks that the column set hasn't changed since the
// last time this file was updated.
checkColumns(Change.Id.class, 1);
checkColumns(
Change.class, 1, 2, 3, 4, 5, 7, 8, 10, 12, 13, 14, 17, 18, 19, 20, 21, 22, 23, 101);
checkColumns(ChangeMessage.Key.class, 1, 2);
checkColumns(ChangeMessage.class, 1, 2, 3, 4, 5, 6, 7);
checkColumns(PatchSet.Id.class, 1, 2);
checkColumns(PatchSet.class, 1, 2, 3, 4, 6, 8, 9);
checkColumns(PatchSetApproval.Key.class, 1, 2, 3);
checkColumns(PatchSetApproval.class, 1, 2, 3, 6, 7, 8);
checkColumns(PatchLineComment.Key.class, 1, 2);
checkColumns(PatchLineComment.class, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
}
private final Change change;
private final ImmutableList<ChangeMessage> changeMessages;
private final ImmutableSortedMap<PatchSet.Id, PatchSet> patchSets;
private final ImmutableMap<PatchSetApproval.Key, PatchSetApproval> patchSetApprovals;
private final ImmutableMap<PatchLineComment.Key, PatchLineComment> patchLineComments;
private final ReviewerSet reviewers;
private final Source source;
public ChangeBundle(
Change change,
Iterable<ChangeMessage> changeMessages,
Iterable<PatchSet> patchSets,
Iterable<PatchSetApproval> patchSetApprovals,
Iterable<PatchLineComment> patchLineComments,
ReviewerSet reviewers,
Source source) {
this.change = requireNonNull(change);
this.changeMessages = changeMessageList(changeMessages);
this.patchSets = ImmutableSortedMap.copyOfSorted(patchSetMap(patchSets));
this.patchSetApprovals = ImmutableMap.copyOf(patchSetApprovalMap(patchSetApprovals));
this.patchLineComments = ImmutableMap.copyOf(patchLineCommentMap(patchLineComments));
this.reviewers = requireNonNull(reviewers);
this.source = requireNonNull(source);
for (ChangeMessage m : this.changeMessages) {
checkArgument(m.getKey().getParentKey().equals(change.getId()));
}
for (PatchSet.Id id : this.patchSets.keySet()) {
checkArgument(id.getParentKey().equals(change.getId()));
}
for (PatchSetApproval.Key k : this.patchSetApprovals.keySet()) {
checkArgument(k.getParentKey().getParentKey().equals(change.getId()));
}
for (PatchLineComment.Key k : this.patchLineComments.keySet()) {
checkArgument(k.getParentKey().getParentKey().getParentKey().equals(change.getId()));
}
}
public Change getChange() {
return change;
}
public ImmutableCollection<ChangeMessage> getChangeMessages() {
return changeMessages;
}
public ImmutableCollection<PatchSet> getPatchSets() {
return patchSets.values();
}
public ImmutableCollection<PatchSetApproval> getPatchSetApprovals() {
return patchSetApprovals.values();
}
public ImmutableCollection<PatchLineComment> getPatchLineComments() {
return patchLineComments.values();
}
public ReviewerSet getReviewers() {
return reviewers;
}
public Source getSource() {
return source;
}
public ImmutableList<String> differencesFrom(ChangeBundle o) {
List<String> diffs = new ArrayList<>();
diffChanges(diffs, this, o);
diffChangeMessages(diffs, this, o);
diffPatchSets(diffs, this, o);
diffPatchSetApprovals(diffs, this, o);
diffReviewers(diffs, this, o);
diffPatchLineComments(diffs, this, o);
return ImmutableList.copyOf(diffs);
}
private Timestamp getFirstPatchSetTime() {
if (patchSets.isEmpty()) {
return change.getCreatedOn();
}
return patchSets.firstEntry().getValue().getCreatedOn();
}
private Timestamp getLatestTimestamp() {
Ordering<Timestamp> o = Ordering.natural().nullsFirst();
Timestamp ts = null;
for (ChangeMessage cm : filterChangeMessages()) {
ts = o.max(ts, cm.getWrittenOn());
}
for (PatchSet ps : getPatchSets()) {
ts = o.max(ts, ps.getCreatedOn());
}
for (PatchSetApproval psa : filterPatchSetApprovals().values()) {
ts = o.max(ts, psa.getGranted());
}
for (PatchLineComment plc : filterPatchLineComments().values()) {
// Ignore draft comments, as they do not show up in the change meta graph.
if (plc.getStatus() != PatchLineComment.Status.DRAFT) {
ts = o.max(ts, plc.getWrittenOn());
}
}
return firstNonNull(ts, change.getLastUpdatedOn());
}
private Map<PatchSetApproval.Key, PatchSetApproval> filterPatchSetApprovals() {
return limitToValidPatchSets(patchSetApprovals, PatchSetApproval.Key::getParentKey);
}
private Map<PatchLineComment.Key, PatchLineComment> filterPatchLineComments() {
return limitToValidPatchSets(patchLineComments, k -> k.getParentKey().getParentKey());
}
private <K, V> Map<K, V> limitToValidPatchSets(Map<K, V> in, Function<K, PatchSet.Id> func) {
return Maps.filterKeys(in, Predicates.compose(validPatchSetPredicate(), func));
}
private Predicate<PatchSet.Id> validPatchSetPredicate() {
return patchSets::containsKey;
}
private Collection<ChangeMessage> filterChangeMessages() {
final Predicate<PatchSet.Id> validPatchSet = validPatchSetPredicate();
return Collections2.filter(
changeMessages,
m -> {
PatchSet.Id psId = m.getPatchSetId();
if (psId == null) {
return true;
}
return validPatchSet.apply(psId);
});
}
private static void diffChanges(List<String> diffs, ChangeBundle bundleA, ChangeBundle bundleB) {
Change a = bundleA.change;
Change b = bundleB.change;
String desc = a.getId().equals(b.getId()) ? describe(a.getId()) : "Changes";
boolean excludeCreatedOn = false;
boolean excludeCurrentPatchSetId = false;
boolean excludeTopic = false;
Timestamp aCreated = a.getCreatedOn();
Timestamp bCreated = b.getCreatedOn();
Timestamp aUpdated = a.getLastUpdatedOn();
Timestamp bUpdated = b.getLastUpdatedOn();
boolean excludeSubject = false;
boolean excludeOrigSubj = false;
// Subject is not technically a nullable field, but we observed some null
// subjects in the wild on googlesource.com, so treat null as empty.
String aSubj = Strings.nullToEmpty(a.getSubject());
String bSubj = Strings.nullToEmpty(b.getSubject());
// Allow created timestamp in NoteDb to be any of:
// - The created timestamp of the change.
// - The timestamp of the first remaining patch set.
// - The last updated timestamp, if it is less than the created timestamp.
//
// Ignore subject if the NoteDb subject starts with the ReviewDb subject.
// The NoteDb subject is read directly from the commit, whereas the ReviewDb
// subject historically may have been truncated to fit in a SQL varchar
// column.
//
// Ignore original subject on the ReviewDb side when comparing to NoteDb.
// This field may have any number of values:
// - It may be null, if the change has had no new patch sets pushed since
// migrating to schema 103.
// - It may match the first patch set subject, if the change was created
// after migrating to schema 103.
// - It may match the subject of the first patch set that was pushed after
// the migration to schema 103, even though that is neither the subject
// of the first patch set nor the subject of the last patch set. (See
// Change#setCurrentPatchSet as of 43b10f86 for this behavior.) This
// subject of an intermediate patch set is not available to the
// ChangeBundle; we would have to get the subject from the repo, which is
// inconvenient at this point.
//
// Ignore original subject on the ReviewDb side if it equals the subject of
// the current patch set.
//
// For all of the above subject comparisons, first trim any leading spaces
// from the NoteDb strings. (We actually do represent the leading spaces
// faithfully during conversion, but JGit's FooterLine parser trims them
// when reading.)
//
// Ignore empty topic on the ReviewDb side if it is null on the NoteDb side.
//
// Ignore currentPatchSetId on NoteDb side if ReviewDb does not point to a
// valid patch set.
//
// Use max timestamp of all ReviewDb entities when comparing with NoteDb.
if (bundleA.source == REVIEW_DB && bundleB.source == NOTE_DB) {
boolean createdOnMatchesFirstPs =
!timestampsDiffer(bundleA, bundleA.getFirstPatchSetTime(), bundleB, bCreated);
boolean createdOnMatchesLastUpdatedOn =
!timestampsDiffer(bundleA, aUpdated, bundleB, bCreated);
boolean createdAfterUpdated = aCreated.compareTo(aUpdated) > 0;
excludeCreatedOn =
createdOnMatchesFirstPs || (createdAfterUpdated && createdOnMatchesLastUpdatedOn);
aSubj = cleanReviewDbSubject(aSubj);
bSubj = cleanNoteDbSubject(bSubj);
excludeCurrentPatchSetId = !bundleA.validPatchSetPredicate().apply(a.currentPatchSetId());
excludeSubject = bSubj.startsWith(aSubj) || excludeCurrentPatchSetId;
excludeOrigSubj = true;
String aTopic = trimOrNull(a.getTopic());
excludeTopic =
Objects.equals(aTopic, b.getTopic()) || ("".equals(aTopic) && b.getTopic() == null);
aUpdated = bundleA.getLatestTimestamp();
} else if (bundleA.source == NOTE_DB && bundleB.source == REVIEW_DB) {
boolean createdOnMatchesFirstPs =
!timestampsDiffer(bundleA, aCreated, bundleB, bundleB.getFirstPatchSetTime());
boolean createdOnMatchesLastUpdatedOn =
!timestampsDiffer(bundleA, aCreated, bundleB, bUpdated);
boolean createdAfterUpdated = bCreated.compareTo(bUpdated) > 0;
excludeCreatedOn =
createdOnMatchesFirstPs || (createdAfterUpdated && createdOnMatchesLastUpdatedOn);
aSubj = cleanNoteDbSubject(aSubj);
bSubj = cleanReviewDbSubject(bSubj);
excludeCurrentPatchSetId = !bundleB.validPatchSetPredicate().apply(b.currentPatchSetId());
excludeSubject = aSubj.startsWith(bSubj) || excludeCurrentPatchSetId;
excludeOrigSubj = true;
String bTopic = trimOrNull(b.getTopic());
excludeTopic =
Objects.equals(bTopic, a.getTopic()) || (a.getTopic() == null && "".equals(bTopic));
bUpdated = bundleB.getLatestTimestamp();
}
String subjectField = "subject";
String updatedField = "lastUpdatedOn";
List<String> exclude =
Lists.newArrayList(subjectField, updatedField, "noteDbState", "rowVersion");
if (excludeCreatedOn) {
exclude.add("createdOn");
}
if (excludeCurrentPatchSetId) {
exclude.add("currentPatchSetId");
}
if (excludeOrigSubj) {
exclude.add("originalSubject");
}
if (excludeTopic) {
exclude.add("topic");
}
diffColumnsExcluding(diffs, Change.class, desc, bundleA, a, bundleB, b, exclude);
// Allow last updated timestamps to either be exactly equal (within slop),
// or the NoteDb timestamp to be equal to the latest entity timestamp in the
// whole ReviewDb bundle (within slop).
if (timestampsDiffer(bundleA, a.getLastUpdatedOn(), bundleB, b.getLastUpdatedOn())) {
diffTimestamps(
diffs, desc, bundleA, aUpdated, bundleB, bUpdated, "effective last updated time");
}
if (!excludeSubject) {
diffValues(diffs, desc, aSubj, bSubj, subjectField);
}
}
private static String trimOrNull(String s) {
return s != null ? CharMatcher.whitespace().trimFrom(s) : null;
}
private static String cleanReviewDbSubject(String s) {
s = CharMatcher.is(' ').trimLeadingFrom(s);
// An old JGit bug failed to extract subjects from commits with "\r\n"
// terminators: https://bugs.eclipse.org/bugs/show_bug.cgi?id=400707
// Changes created with this bug may have "\r\n" converted to "\r " and the
// entire commit in the subject. The version of JGit used to read NoteDb
// changes parses these subjects correctly, so we need to clean up old
// ReviewDb subjects before comparing.
int rn = s.indexOf("\r \r ");
if (rn >= 0) {
s = s.substring(0, rn);
}
return NoteDbUtil.sanitizeFooter(s);
}
private static String cleanNoteDbSubject(String s) {
return NoteDbUtil.sanitizeFooter(s);
}
/**
* Set of fields that must always exactly match between ReviewDb and NoteDb.
*
* <p>Used to limit the worst-case quadratic search when pairing off matching messages below.
*/
@AutoValue
abstract static class ChangeMessageCandidate {
static ChangeMessageCandidate create(ChangeMessage cm) {
return new AutoValue_ChangeBundle_ChangeMessageCandidate(
cm.getAuthor(), cm.getMessage(), cm.getTag());
}
@Nullable
abstract Account.Id author();
@Nullable
abstract String message();
@Nullable
abstract String tag();
// Exclude:
// - patch set, which may be null on ReviewDb side but not NoteDb
// - UUID, which is always different between ReviewDb and NoteDb
// - writtenOn, which is fuzzy
}
private static void diffChangeMessages(
List<String> diffs, ChangeBundle bundleA, ChangeBundle bundleB) {
if (bundleA.source == REVIEW_DB && bundleB.source == REVIEW_DB) {
// Both came from ReviewDb: check all fields exactly.
Map<ChangeMessage.Key, ChangeMessage> as = changeMessageMap(bundleA.filterChangeMessages());
Map<ChangeMessage.Key, ChangeMessage> bs = changeMessageMap(bundleB.filterChangeMessages());
for (ChangeMessage.Key k : diffKeySets(diffs, as, bs)) {
ChangeMessage a = as.get(k);
ChangeMessage b = bs.get(k);
String desc = describe(k);
diffColumns(diffs, ChangeMessage.class, desc, bundleA, a, bundleB, b);
}
return;
}
Change.Id id = bundleA.getChange().getId();
checkArgument(id.equals(bundleB.getChange().getId()));
// Try to pair up matching ChangeMessages from each side, and succeed only
// if both collections are empty at the end. Quadratic in the worst case,
// but easy to reason about.
List<ChangeMessage> as = new LinkedList<>(bundleA.filterChangeMessages());
ListMultimap<ChangeMessageCandidate, ChangeMessage> bs = LinkedListMultimap.create();
for (ChangeMessage b : bundleB.filterChangeMessages()) {
bs.put(ChangeMessageCandidate.create(b), b);
}
Iterator<ChangeMessage> ait = as.iterator();
A:
while (ait.hasNext()) {
ChangeMessage a = ait.next();
Iterator<ChangeMessage> bit = bs.get(ChangeMessageCandidate.create(a)).iterator();
while (bit.hasNext()) {
ChangeMessage b = bit.next();
if (changeMessagesMatch(bundleA, a, bundleB, b)) {
ait.remove();
bit.remove();
continue A;
}
}
}
if (as.isEmpty() && bs.isEmpty()) {
return;
}
StringBuilder sb =
new StringBuilder("ChangeMessages differ for Change.Id ").append(id).append('\n');
if (!as.isEmpty()) {
sb.append("Only in A:");
for (ChangeMessage cm : as) {
sb.append("\n ").append(cm);
}
if (!bs.isEmpty()) {
sb.append('\n');
}
}
if (!bs.isEmpty()) {
sb.append("Only in B:");
bs.values()
.stream()
.sorted(CHANGE_MESSAGE_COMPARATOR)
.forEach(cm -> sb.append("\n ").append(cm));
}
diffs.add(sb.toString());
}
private static boolean changeMessagesMatch(
ChangeBundle bundleA, ChangeMessage a, ChangeBundle bundleB, ChangeMessage b) {
List<String> tempDiffs = new ArrayList<>();
String temp = "temp";
// ReviewDb allows timestamps before patch set was created, but NoteDb
// truncates this to the patch set creation timestamp.
Timestamp ta = a.getWrittenOn();
Timestamp tb = b.getWrittenOn();
PatchSet psa = bundleA.patchSets.get(a.getPatchSetId());
PatchSet psb = bundleB.patchSets.get(b.getPatchSetId());
boolean excludePatchSet = false;
boolean excludeWrittenOn = false;
if (bundleA.source == REVIEW_DB && bundleB.source == NOTE_DB) {
excludePatchSet = a.getPatchSetId() == null;
excludeWrittenOn =
psa != null
&& psb != null
&& ta.before(psa.getCreatedOn())
&& tb.equals(psb.getCreatedOn());
} else if (bundleA.source == NOTE_DB && bundleB.source == REVIEW_DB) {
excludePatchSet = b.getPatchSetId() == null;
excludeWrittenOn =
psa != null
&& psb != null
&& tb.before(psb.getCreatedOn())
&& ta.equals(psa.getCreatedOn());
}
List<String> exclude = Lists.newArrayList("key");
if (excludePatchSet) {
exclude.add("patchset");
}
if (excludeWrittenOn) {
exclude.add("writtenOn");
}
diffColumnsExcluding(tempDiffs, ChangeMessage.class, temp, bundleA, a, bundleB, b, exclude);
return tempDiffs.isEmpty();
}
private static void diffPatchSets(
List<String> diffs, ChangeBundle bundleA, ChangeBundle bundleB) {
Map<PatchSet.Id, PatchSet> as = bundleA.patchSets;
Map<PatchSet.Id, PatchSet> bs = bundleB.patchSets;
Optional<PatchSet.Id> minA = as.keySet().stream().min(intKeyOrdering());
Optional<PatchSet.Id> minB = bs.keySet().stream().min(intKeyOrdering());
Set<PatchSet.Id> ids = diffKeySets(diffs, as, bs);
// Old versions of Gerrit had a bug that created patch sets during
// rebase or submission with a createdOn timestamp earlier than the patch
// set it was replacing. (In the cases I examined, it was equal to createdOn
// for the change, but we're not counting on this exact behavior.)
//
// ChangeRebuilder ensures patch set events come out in order, but it's hard
// to predict what the resulting timestamps would look like. So, completely
// ignore the createdOn timestamps if both:
// * ReviewDb timestamps are non-monotonic.
// * NoteDb timestamps are monotonic.
//
// Allow the timestamp of the first patch set to match the creation time of
// the change.
boolean excludeAllCreatedOn = false;
if (bundleA.source == REVIEW_DB && bundleB.source == NOTE_DB) {
excludeAllCreatedOn = !createdOnIsMonotonic(as, ids) && createdOnIsMonotonic(bs, ids);
} else if (bundleA.source == NOTE_DB && bundleB.source == REVIEW_DB) {
excludeAllCreatedOn = createdOnIsMonotonic(as, ids) && !createdOnIsMonotonic(bs, ids);
}
for (PatchSet.Id id : ids) {
PatchSet a = as.get(id);
PatchSet b = bs.get(id);
String desc = describe(id);
String pushCertField = "pushCertificate";
boolean excludeCreatedOn = excludeAllCreatedOn;
boolean excludeDesc = false;
if (bundleA.source == REVIEW_DB && bundleB.source == NOTE_DB) {
excludeDesc = Objects.equals(trimOrNull(a.getDescription()), b.getDescription());
excludeCreatedOn |=
Optional.of(id).equals(minB) && b.getCreatedOn().equals(bundleB.change.getCreatedOn());
} else if (bundleA.source == NOTE_DB && bundleB.source == REVIEW_DB) {
excludeDesc = Objects.equals(a.getDescription(), trimOrNull(b.getDescription()));
excludeCreatedOn |=
Optional.of(id).equals(minA) && a.getCreatedOn().equals(bundleA.change.getCreatedOn());
}
List<String> exclude = Lists.newArrayList(pushCertField);
if (excludeCreatedOn) {
exclude.add("createdOn");
}
if (excludeDesc) {
exclude.add("description");
}
diffColumnsExcluding(diffs, PatchSet.class, desc, bundleA, a, bundleB, b, exclude);
diffValues(diffs, desc, trimPushCert(a), trimPushCert(b), pushCertField);
}
}
private static String trimPushCert(PatchSet ps) {
if (ps.getPushCertificate() == null) {
return null;
}
return CharMatcher.is('\n').trimTrailingFrom(ps.getPushCertificate());
}
private static boolean createdOnIsMonotonic(
Map<?, PatchSet> patchSets, Set<PatchSet.Id> limitToIds) {
List<PatchSet> orderedById =
patchSets
.values()
.stream()
.filter(ps -> limitToIds.contains(ps.getId()))
.sorted(ChangeUtil.PS_ID_ORDER)
.collect(toList());
return Ordering.natural().onResultOf(PatchSet::getCreatedOn).isOrdered(orderedById);
}
private static void diffPatchSetApprovals(
List<String> diffs, ChangeBundle bundleA, ChangeBundle bundleB) {
Map<PatchSetApproval.Key, PatchSetApproval> as = bundleA.filterPatchSetApprovals();
Map<PatchSetApproval.Key, PatchSetApproval> bs = bundleB.filterPatchSetApprovals();
for (PatchSetApproval.Key k : diffKeySets(diffs, as, bs)) {
PatchSetApproval a = as.get(k);
PatchSetApproval b = bs.get(k);
String desc = describe(k);
// ReviewDb allows timestamps before patch set was created, but NoteDb
// truncates this to the patch set creation timestamp.
//
// ChangeRebuilder ensures all post-submit approvals happen after the
// actual submit, so the timestamps may not line up. This shouldn't really
// happen, because postSubmit shouldn't be set in ReviewDb until after the
// change is submitted in ReviewDb, but you never know.
//
// Due to a quirk of PostReview, post-submit 0 votes might not have the
// postSubmit bit set in ReviewDb. As these are only used for tombstone
// purposes, ignore the postSubmit bit in NoteDb in this case.
Timestamp ta = a.getGranted();
Timestamp tb = b.getGranted();
PatchSet psa = requireNonNull(bundleA.patchSets.get(a.getPatchSetId()));
PatchSet psb = requireNonNull(bundleB.patchSets.get(b.getPatchSetId()));
boolean excludeGranted = false;
boolean excludePostSubmit = false;
List<String> exclude = new ArrayList<>(1);
if (bundleA.source == REVIEW_DB && bundleB.source == NOTE_DB) {
excludeGranted =
(ta.before(psa.getCreatedOn()) && tb.equals(psb.getCreatedOn()))
|| ta.compareTo(tb) < 0;
excludePostSubmit = a.getValue() == 0 && b.isPostSubmit();
} else if (bundleA.source == NOTE_DB && bundleB.source == REVIEW_DB) {
excludeGranted =
(tb.before(psb.getCreatedOn()) && ta.equals(psa.getCreatedOn()))
|| (tb.compareTo(ta) < 0);
excludePostSubmit = b.getValue() == 0 && a.isPostSubmit();
}
// Legacy submit approvals may or may not have tags associated with them,
// depending on whether ChangeRebuilder happened to group them with the
// status change.
boolean excludeTag =
bundleA.source != bundleB.source && a.isLegacySubmit() && b.isLegacySubmit();
if (excludeGranted) {
exclude.add("granted");
}
if (excludePostSubmit) {
exclude.add("postSubmit");
}
if (excludeTag) {
exclude.add("tag");
}
diffColumnsExcluding(diffs, PatchSetApproval.class, desc, bundleA, a, bundleB, b, exclude);
}
}
private static void diffReviewers(
List<String> diffs, ChangeBundle bundleA, ChangeBundle bundleB) {
diffSets(diffs, bundleA.reviewers.all(), bundleB.reviewers.all(), "reviewer");
}
private static void diffPatchLineComments(
List<String> diffs, ChangeBundle bundleA, ChangeBundle bundleB) {
Map<PatchLineComment.Key, PatchLineComment> as = bundleA.filterPatchLineComments();
Map<PatchLineComment.Key, PatchLineComment> bs = bundleB.filterPatchLineComments();
for (PatchLineComment.Key k : diffKeySets(diffs, as, bs)) {
PatchLineComment a = as.get(k);
PatchLineComment b = bs.get(k);
String desc = describe(k);
diffColumns(diffs, PatchLineComment.class, desc, bundleA, a, bundleB, b);
}
}
private static <T> Set<T> diffKeySets(List<String> diffs, Map<T, ?> a, Map<T, ?> b) {
if (a.isEmpty() && b.isEmpty()) {
return a.keySet();
}
String clazz = keyClass((!a.isEmpty() ? a.keySet() : b.keySet()).iterator().next());
return diffSets(diffs, a.keySet(), b.keySet(), clazz);
}
private static <T> Set<T> diffSets(List<String> diffs, Set<T> as, Set<T> bs, String desc) {
if (as.isEmpty() && bs.isEmpty()) {
return as;
}
Set<T> aNotB = Sets.difference(as, bs);
Set<T> bNotA = Sets.difference(bs, as);
if (aNotB.isEmpty() && bNotA.isEmpty()) {
return as;
}
diffs.add(desc + " sets differ: " + aNotB + " only in A; " + bNotA + " only in B");
return Sets.intersection(as, bs);
}
private static <T> void diffColumns(
List<String> diffs,
Class<T> clazz,
String desc,
ChangeBundle bundleA,
T a,
ChangeBundle bundleB,
T b) {
diffColumnsExcluding(diffs, clazz, desc, bundleA, a, bundleB, b);
}
private static <T> void diffColumnsExcluding(
List<String> diffs,
Class<T> clazz,
String desc,
ChangeBundle bundleA,
T a,
ChangeBundle bundleB,
T b,
String... exclude) {
diffColumnsExcluding(diffs, clazz, desc, bundleA, a, bundleB, b, Arrays.asList(exclude));
}
private static <T> void diffColumnsExcluding(
List<String> diffs,
Class<T> clazz,
String desc,
ChangeBundle bundleA,
T a,
ChangeBundle bundleB,
T b,
Iterable<String> exclude) {
Set<String> toExclude = Sets.newLinkedHashSet(exclude);
for (Field f : clazz.getDeclaredFields()) {
Column col = f.getAnnotation(Column.class);
if (col == null) {
continue;
} else if (toExclude.remove(f.getName())) {
continue;
}
f.setAccessible(true);
try {
if (Timestamp.class.isAssignableFrom(f.getType())) {
diffTimestamps(diffs, desc, bundleA, a, bundleB, b, f.getName());
} else {
diffValues(diffs, desc, f.get(a), f.get(b), f.getName());
}
} catch (IllegalAccessException e) {
throw new IllegalArgumentException(e);
}
}
checkArgument(
toExclude.isEmpty(),
"requested columns to exclude not present in %s: %s",
clazz.getSimpleName(),
toExclude);
}
private static void diffTimestamps(
List<String> diffs,
String desc,
ChangeBundle bundleA,
Object a,
ChangeBundle bundleB,
Object b,
String field) {
checkArgument(a.getClass() == b.getClass());
Class<?> clazz = a.getClass();
Timestamp ta;
Timestamp tb;
try {
Field f = clazz.getDeclaredField(field);
checkArgument(f.getAnnotation(Column.class) != null);
f.setAccessible(true);
ta = (Timestamp) f.get(a);
tb = (Timestamp) f.get(b);
} catch (IllegalAccessException | NoSuchFieldException | SecurityException e) {
throw new IllegalArgumentException(e);
}
diffTimestamps(diffs, desc, bundleA, ta, bundleB, tb, field);
}
private static void diffTimestamps(
List<String> diffs,
String desc,
ChangeBundle bundleA,
Timestamp ta,
ChangeBundle bundleB,
Timestamp tb,
String fieldDesc) {
if (bundleA.source == bundleB.source || ta == null || tb == null) {
diffValues(diffs, desc, ta, tb, fieldDesc);
} else if (bundleA.source == NOTE_DB) {
diffTimestamps(diffs, desc, bundleA.getChange(), ta, bundleB.getChange(), tb, fieldDesc);
} else {
diffTimestamps(diffs, desc, bundleB.getChange(), tb, bundleA.getChange(), ta, fieldDesc);
}
}
private static boolean timestampsDiffer(
ChangeBundle bundleA, Timestamp ta, ChangeBundle bundleB, Timestamp tb) {
List<String> tempDiffs = new ArrayList<>(1);
diffTimestamps(tempDiffs, "temp", bundleA, ta, bundleB, tb, "temp");
return !tempDiffs.isEmpty();
}
private static void diffTimestamps(
List<String> diffs,
String desc,
Change changeFromNoteDb,
Timestamp tsFromNoteDb,
Change changeFromReviewDb,
Timestamp tsFromReviewDb,
String field) {
// Because ChangeRebuilder may batch events together that are several
// seconds apart, the timestamp in NoteDb may actually be several seconds
// *earlier* than the timestamp in ReviewDb that it was converted from.
checkArgument(
tsFromNoteDb.equals(truncateToSecond(tsFromNoteDb)),
"%s from NoteDb has non-rounded %s timestamp: %s",
desc,
field,
tsFromNoteDb);
if (tsFromReviewDb.before(changeFromReviewDb.getCreatedOn())
&& tsFromNoteDb.equals(changeFromNoteDb.getCreatedOn())) {
// Timestamp predates change creation. These are truncated to change
// creation time during NoteDb conversion, so allow this if the timestamp
// in NoteDb matches the createdOn time in NoteDb.
return;
}
long delta = tsFromReviewDb.getTime() - tsFromNoteDb.getTime();
long max = ChangeRebuilderImpl.MAX_WINDOW_MS;
if (delta < 0 || delta > max) {
diffs.add(
field
+ " differs for "
+ desc
+ " in NoteDb vs. ReviewDb:"
+ " {"
+ tsFromNoteDb
+ "} != {"
+ tsFromReviewDb
+ "}");
}
}
private static void diffValues(
List<String> diffs, String desc, Object va, Object vb, String name) {
if (!Objects.equals(va, vb)) {
diffs.add(name + " differs for " + desc + ": {" + va + "} != {" + vb + "}");
}
}
private static String describe(Object key) {
return keyClass(key) + " " + key;
}
private static String keyClass(Object obj) {
Class<?> clazz = obj.getClass();
String name = clazz.getSimpleName();
checkArgument(name.endsWith("Key") || name.endsWith("Id"), "not an Id/Key class: %s", name);
if (name.equals("Key") || name.equals("Id")) {
return clazz.getEnclosingClass().getSimpleName() + "." + name;
} else if (name.startsWith("AutoValue_")) {
return name.substring(name.lastIndexOf('_') + 1);
}
return name;
}
@Override
public String toString() {
return getClass().getSimpleName()
+ "{id="
+ change.getId()
+ ", ChangeMessage["
+ changeMessages.size()
+ "]"
+ ", PatchSet["
+ patchSets.size()
+ "]"
+ ", PatchSetApproval["
+ patchSetApprovals.size()
+ "]"
+ ", PatchLineComment["
+ patchLineComments.size()
+ "]"
+ "}";
}
}

25
java/com/google/gerrit/server/notedb/ChangeBundleReader.java
View File

@@ -1,25 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gwtorm.server.OrmException;
public interface ChangeBundleReader {
@Nullable
ChangeBundle fromReviewDb(ReviewDb db, Change.Id id) throws OrmException;
}

52
java/com/google/gerrit/server/notedb/GwtormChangeBundleReader.java
View File

@@ -1,52 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.PatchSetApproval;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.ReviewerSet;
import com.google.gerrit.server.notedb.ChangeBundle.Source;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.util.List;
@Singleton
public class GwtormChangeBundleReader implements ChangeBundleReader {
@Inject
GwtormChangeBundleReader() {}
@Override
@Nullable
public ChangeBundle fromReviewDb(ReviewDb db, Change.Id id) throws OrmException {
Change reviewDbChange = db.changes().get(id);
if (reviewDbChange == null) {
return null;
}
// TODO(dborowitz): Figure out how to do this more consistently, e.g. hand-written inner joins.
List<PatchSetApproval> approvals = db.patchSetApprovals().byChange(id).toList();
return new ChangeBundle(
reviewDbChange,
db.changeMessages().byChange(id),
db.patchSets().byChange(id),
approvals,
db.patchComments().byChange(id),
ReviewerSet.fromApprovals(approvals),
Source.REVIEW_DB);
}
}

66
java/com/google/gerrit/server/notedb/NoteDbModule.java
View File

@@ -17,31 +17,21 @@ package com.google.gerrit.server.notedb;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.gerrit.extensions.config.FactoryModule;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Change.Id;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.notedb.NoteDbUpdateManager.Result;
import com.google.gerrit.server.notedb.rebuild.ChangeRebuilder;
import com.google.gerrit.server.notedb.rebuild.ChangeRebuilderImpl;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Names;
import org.eclipse.jgit.lib.Config;
public class NoteDbModule extends FactoryModule {
private final Config cfg;
private final boolean useTestBindings;
static NoteDbModule forTest(Config cfg) {
return new NoteDbModule(cfg, true);
static NoteDbModule forTest() {
return new NoteDbModule(true);
}
public NoteDbModule(Config cfg) {
this(cfg, false);
public NoteDbModule() {
this(false);
}
private NoteDbModule(Config cfg, boolean useTestBindings) {
this.cfg = cfg;
private NoteDbModule(boolean useTestBindings) {
this.useTestBindings = useTestBindings;
}
@@ -57,53 +47,7 @@ public class NoteDbModule extends FactoryModule {
if (!useTestBindings) {
install(ChangeNotesCache.module());
if (cfg.getBoolean("noteDb", null, "testRebuilderWrapper", false)) {
// Yes, another variety of test bindings with a different way of
// configuring it.
bind(ChangeRebuilder.class).to(TestChangeRebuilderWrapper.class);
} else {
bind(ChangeRebuilder.class).to(ChangeRebuilderImpl.class);
}
} else {
bind(ChangeRebuilder.class)
.toInstance(
new ChangeRebuilder(null) {
@Override
public Result rebuild(ReviewDb db, Change.Id changeId) {
return null;
}
@Override
public Result rebuildEvenIfReadOnly(ReviewDb db, Id changeId) {
return null;
}
@Override
public Result rebuild(NoteDbUpdateManager manager, ChangeBundle bundle) {
return null;
}
@Override
public NoteDbUpdateManager stage(ReviewDb db, Change.Id changeId) {
return null;
}
@Override
public Result execute(
ReviewDb db, Change.Id changeId, NoteDbUpdateManager manager) {
return null;
}
@Override
public void buildUpdates(NoteDbUpdateManager manager, ChangeBundle bundle) {
// Do nothing.
}
@Override
public void rebuildReviewDb(ReviewDb db, Project.NameKey project, Id changeId) {
// Do nothing.
}
});
bind(new TypeLiteral<Cache<ChangeNotesCache.Key, ChangeNotesState>>() {})
.annotatedWith(Names.named(ChangeNotesCache.CACHE_NAME))
.toInstance(CacheBuilder.newBuilder().<ChangeNotesCache.Key, ChangeNotesState>build());

125
java/com/google/gerrit/server/notedb/TestChangeRebuilderWrapper.java
View File

@@ -1,125 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb;
import com.google.common.annotations.VisibleForTesting;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Change.Id;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.notedb.NoteDbUpdateManager.Result;
import com.google.gerrit.server.notedb.rebuild.ChangeRebuilder;
import com.google.gerrit.server.notedb.rebuild.ChangeRebuilderImpl;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
@VisibleForTesting
@Singleton
public class TestChangeRebuilderWrapper extends ChangeRebuilder {
private final ChangeRebuilderImpl delegate;
private final AtomicBoolean failNextUpdate;
private final AtomicBoolean stealNextUpdate;
@Inject
TestChangeRebuilderWrapper(SchemaFactory<ReviewDb> schemaFactory, ChangeRebuilderImpl rebuilder) {
super(schemaFactory);
this.delegate = rebuilder;
this.failNextUpdate = new AtomicBoolean();
this.stealNextUpdate = new AtomicBoolean();
}
public void failNextUpdate() {
failNextUpdate.set(true);
}
public void stealNextUpdate() {
stealNextUpdate.set(true);
}
@Override
public Result rebuild(ReviewDb db, Change.Id changeId) throws IOException, OrmException {
return rebuild(db, changeId, true);
}
@Override
public Result rebuildEvenIfReadOnly(ReviewDb db, Change.Id changeId)
throws IOException, OrmException {
return rebuild(db, changeId, false);
}
private Result rebuild(ReviewDb db, Change.Id changeId, boolean checkReadOnly)
throws IOException, OrmException {
if (failNextUpdate.getAndSet(false)) {
throw new IOException("Update failed");
}
Result result =
checkReadOnly
? delegate.rebuild(db, changeId)
: delegate.rebuildEvenIfReadOnly(db, changeId);
if (stealNextUpdate.getAndSet(false)) {
throw new IOException("Update stolen");
}
return result;
}
@Override
public Result rebuild(NoteDbUpdateManager manager, ChangeBundle bundle)
throws IOException, OrmException {
// stealNextUpdate doesn't really apply in this case because the IOException
// would normally come from the manager.execute() method, which isn't called
// here.
return delegate.rebuild(manager, bundle);
}
@Override
public NoteDbUpdateManager stage(ReviewDb db, Change.Id changeId)
throws IOException, OrmException {
// Don't inspect stealNextUpdate; that happens in execute() below.
return delegate.stage(db, changeId);
}
@Override
public Result execute(ReviewDb db, Change.Id changeId, NoteDbUpdateManager manager)
throws OrmException, IOException {
if (failNextUpdate.getAndSet(false)) {
throw new IOException("Update failed");
}
Result result = delegate.execute(db, changeId, manager);
if (stealNextUpdate.getAndSet(false)) {
throw new IOException("Update stolen");
}
return result;
}
@Override
public void buildUpdates(NoteDbUpdateManager manager, ChangeBundle bundle)
throws IOException, OrmException {
// Don't check for manual failure; that happens in execute().
delegate.buildUpdates(manager, bundle);
}
@Override
public void rebuildReviewDb(ReviewDb db, Project.NameKey project, Id changeId)
throws OrmException {
if (failNextUpdate.getAndSet(false)) {
throw new OrmException("Update failed");
}
delegate.rebuildReviewDb(db, project, changeId);
}
}

25
java/com/google/gerrit/server/notedb/rebuild/AbortUpdateException.java
View File

@@ -1,25 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.gwtorm.server.OrmRuntimeException;
class AbortUpdateException extends OrmRuntimeException {
private static final long serialVersionUID = 1L;
AbortUpdateException() {
super("aborted");
}
}

64
java/com/google/gerrit/server/notedb/rebuild/ApprovalEvent.java
View File

@@ -1,64 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.gerrit.reviewdb.client.PatchSetApproval;
import com.google.gerrit.server.notedb.ChangeUpdate;
import java.sql.Timestamp;
class ApprovalEvent extends Event {
private PatchSetApproval psa;
ApprovalEvent(PatchSetApproval psa, Timestamp changeCreatedOn) {
super(
psa.getPatchSetId(),
psa.getAccountId(),
psa.getRealAccountId(),
psa.getGranted(),
changeCreatedOn,
psa.getTag());
this.psa = psa;
}
@Override
boolean uniquePerUpdate() {
return false;
}
@Override
protected boolean canHaveTag() {
// Legacy SUBM approvals don't have a tag field set, but the corresponding
// ChangeMessage for merging the change does. We need to let these be in the
// same meta commit so the SUBM approval isn't counted as post-submit.
return !psa.isLegacySubmit();
}
@Override
void apply(ChangeUpdate update) {
checkUpdate(update);
update.putApproval(psa.getLabel(), psa.getValue());
}
@Override
protected boolean isPostSubmitApproval() {
return psa.isPostSubmit();
}
@Override
protected void addToString(ToStringHelper helper) {
helper.add("approval", psa);
}
}

185
java/com/google/gerrit/server/notedb/rebuild/ChangeMessageEvent.java
View File

@@ -1,185 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.ChangeMessage;
import com.google.gerrit.server.ChangeMessagesUtil;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gwtorm.server.OrmException;
import java.sql.Timestamp;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
class ChangeMessageEvent extends Event {
private static final ImmutableMap<Change.Status, Pattern> STATUS_PATTERNS =
ImmutableMap.of(
Change.Status.ABANDONED, Pattern.compile("^Abandoned(\n.*)*$"),
Change.Status.MERGED,
Pattern.compile(
"^Change has been successfully (merged|cherry-picked|rebased|pushed).*$"),
Change.Status.NEW, Pattern.compile("^Restored(\n.*)*$"));
private static final Pattern PRIVATE_SET_REGEXP = Pattern.compile("^Set private$");
private static final Pattern PRIVATE_UNSET_REGEXP = Pattern.compile("^Unset private$");
private static final Pattern TOPIC_SET_REGEXP = Pattern.compile("^Topic set to (.+)$");
private static final Pattern TOPIC_CHANGED_REGEXP =
Pattern.compile("^Topic changed from (.+) to (.+)$");
private static final Pattern TOPIC_REMOVED_REGEXP = Pattern.compile("^Topic (.+) removed$");
private static final Pattern WIP_SET_REGEXP = Pattern.compile("^Set Work In Progress$");
private static final Pattern WIP_UNSET_REGEXP = Pattern.compile("^Set Ready For Review$");
private final Change change;
private final Change noteDbChange;
private final Optional<Change.Status> status;
private final ChangeMessage message;
ChangeMessageEvent(
Change change, Change noteDbChange, ChangeMessage message, Timestamp changeCreatedOn) {
super(
message.getPatchSetId(),
message.getAuthor(),
message.getRealAuthor(),
message.getWrittenOn(),
changeCreatedOn,
message.getTag());
this.change = change;
this.noteDbChange = noteDbChange;
this.message = message;
this.status = parseStatus(message);
}
@Override
boolean uniquePerUpdate() {
return true;
}
@Override
protected boolean isSubmit() {
return status.isPresent() && status.get() == Change.Status.MERGED;
}
@Override
protected boolean canHaveTag() {
return true;
}
@SuppressWarnings("deprecation")
@Override
void apply(ChangeUpdate update) throws OrmException {
checkUpdate(update);
update.setChangeMessage(message.getMessage());
setPrivate(update);
setTopic(update);
setWorkInProgress(update);
if (status.isPresent()) {
Change.Status s = status.get();
update.fixStatus(s);
noteDbChange.setStatus(s);
if (s == Change.Status.MERGED) {
update.setSubmissionId(change.getSubmissionId());
noteDbChange.setSubmissionId(change.getSubmissionId());
}
}
}
private static Optional<Change.Status> parseStatus(ChangeMessage message) {
String msg = message.getMessage();
if (msg == null) {
return Optional.empty();
}
for (Map.Entry<Change.Status, Pattern> e : STATUS_PATTERNS.entrySet()) {
if (e.getValue().matcher(msg).matches()) {
return Optional.of(e.getKey());
}
}
return Optional.empty();
}
private void setPrivate(ChangeUpdate update) {
String msg = message.getMessage();
if (msg == null) {
return;
}
Matcher m = PRIVATE_SET_REGEXP.matcher(msg);
if (m.matches()) {
update.setPrivate(true);
noteDbChange.setPrivate(true);
return;
}
m = PRIVATE_UNSET_REGEXP.matcher(msg);
if (m.matches()) {
update.setPrivate(false);
noteDbChange.setPrivate(false);
}
}
private void setTopic(ChangeUpdate update) {
String msg = message.getMessage();
if (msg == null) {
return;
}
Matcher m = TOPIC_SET_REGEXP.matcher(msg);
if (m.matches()) {
String topic = m.group(1);
update.setTopic(topic);
noteDbChange.setTopic(topic);
return;
}
m = TOPIC_CHANGED_REGEXP.matcher(msg);
if (m.matches()) {
String topic = m.group(2);
update.setTopic(topic);
noteDbChange.setTopic(topic);
return;
}
if (TOPIC_REMOVED_REGEXP.matcher(msg).matches()) {
update.setTopic(null);
noteDbChange.setTopic(null);
}
}
private void setWorkInProgress(ChangeUpdate update) {
String msg = Strings.nullToEmpty(message.getMessage());
String tag = message.getTag();
if (ChangeMessagesUtil.TAG_SET_WIP.equals(tag)
|| ChangeMessagesUtil.TAG_UPLOADED_WIP_PATCH_SET.equals(tag)
|| WIP_SET_REGEXP.matcher(msg).matches()) {
update.setWorkInProgress(true);
noteDbChange.setWorkInProgress(true);
} else if (ChangeMessagesUtil.TAG_SET_READY.equals(tag)
|| ChangeMessagesUtil.TAG_UPLOADED_PATCH_SET.equals(tag)
|| WIP_UNSET_REGEXP.matcher(msg).matches()) {
update.setWorkInProgress(false);
noteDbChange.setWorkInProgress(false);
}
}
@Override
protected void addToString(ToStringHelper helper) {
helper.add("message", message);
}
}

81
java/com/google/gerrit/server/notedb/rebuild/ChangeRebuilder.java
View File

@@ -1,81 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.notedb.ChangeBundle;
import com.google.gerrit.server.notedb.NoteDbUpdateManager;
import com.google.gerrit.server.notedb.NoteDbUpdateManager.Result;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
import java.io.IOException;
public abstract class ChangeRebuilder {
public static class NoPatchSetsException extends OrmException {
private static final long serialVersionUID = 1L;
NoPatchSetsException(Change.Id changeId) {
super("Change " + changeId + " cannot be rebuilt because it has no patch sets");
}
}
private final SchemaFactory<ReviewDb> schemaFactory;
protected ChangeRebuilder(SchemaFactory<ReviewDb> schemaFactory) {
this.schemaFactory = schemaFactory;
}
public final ListenableFuture<Result> rebuildAsync(
Change.Id id, ListeningExecutorService executor) {
return executor.submit(
() -> {
try (ReviewDb db = schemaFactory.open()) {
return rebuild(db, id);
}
});
}
/**
* Rebuild ReviewDb contents by copying from NoteDb.
*
* <p>Requires NoteDb to be the primary storage for the change.
*/
public abstract void rebuildReviewDb(ReviewDb db, Project.NameKey project, Change.Id changeId)
throws OrmException;
// In the following methods "rebuilding" always refers to copying the state
// from ReviewDb to NoteDb, i.e. assuming ReviewDb is the primary storage.
public abstract Result rebuild(ReviewDb db, Change.Id changeId) throws IOException, OrmException;
public abstract Result rebuildEvenIfReadOnly(ReviewDb db, Change.Id changeId)
throws IOException, OrmException;
public abstract Result rebuild(NoteDbUpdateManager manager, ChangeBundle bundle)
throws IOException, OrmException;
public abstract void buildUpdates(NoteDbUpdateManager manager, ChangeBundle bundle)
throws IOException, OrmException;
public abstract NoteDbUpdateManager stage(ReviewDb db, Change.Id changeId)
throws IOException, OrmException;
public abstract Result execute(ReviewDb db, Change.Id changeId, NoteDbUpdateManager manager)
throws OrmException, IOException;
}

687
java/com/google/gerrit/server/notedb/rebuild/ChangeRebuilderImpl.java
View File

@@ -1,687 +0,0 @@
// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.gerrit.reviewdb.client.RefNames.changeMetaRef;
import static com.google.gerrit.server.notedb.ChangeNoteUtil.FOOTER_HASHTAGS;
import static com.google.gerrit.server.notedb.ChangeNoteUtil.FOOTER_PATCH_SET;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import com.google.common.base.Splitter;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.google.common.collect.Table;
import com.google.common.primitives.Ints;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.ChangeMessage;
import com.google.gerrit.reviewdb.client.Comment;
import com.google.gerrit.reviewdb.client.PatchLineComment;
import com.google.gerrit.reviewdb.client.PatchLineComment.Status;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.PatchSetApproval;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.reviewdb.server.ReviewDbUtil;
import com.google.gerrit.server.CommentsUtil;
import com.google.gerrit.server.GerritPersonIdent;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.GerritServerId;
import com.google.gerrit.server.notedb.ChangeBundle;
import com.google.gerrit.server.notedb.ChangeBundleReader;
import com.google.gerrit.server.notedb.ChangeDraftUpdate;
import com.google.gerrit.server.notedb.ChangeNoteUtil;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gerrit.server.notedb.NoteDbChangeState;
import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
import com.google.gerrit.server.notedb.NoteDbUpdateManager;
import com.google.gerrit.server.notedb.NoteDbUpdateManager.OpenRepo;
import com.google.gerrit.server.notedb.NoteDbUpdateManager.Result;
import com.google.gerrit.server.notedb.NotesMigration;
import com.google.gerrit.server.notedb.ReviewerStateInternal;
import com.google.gerrit.server.patch.PatchListCache;
import com.google.gerrit.server.project.NoSuchChangeException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.update.ChainedReceiveCommands;
import com.google.gwtorm.client.Key;
import com.google.gwtorm.server.Access;
import com.google.gwtorm.server.AtomicUpdate;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Inject;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.ReceiveCommand;
public class ChangeRebuilderImpl extends ChangeRebuilder {
/**
* The maximum amount of time between the ReviewDb timestamp of the first and last events batched
* together into a single NoteDb update.
*
* <p>Used to account for the fact that different records with their own timestamps (e.g. {@link
* PatchSetApproval} and {@link ChangeMessage}) historically didn't necessarily use the same
* timestamp, and tended to call {@code System.currentTimeMillis()} independently.
*/
public static final long MAX_WINDOW_MS = SECONDS.toMillis(3);
/**
* The maximum amount of time between two consecutive events to consider them to be in the same
* batch.
*/
static final long MAX_DELTA_MS = SECONDS.toMillis(1);
private final ChangeBundleReader bundleReader;
private final ChangeDraftUpdate.Factory draftUpdateFactory;
private final ChangeNoteUtil changeNoteUtil;
private final ChangeNotes.Factory notesFactory;
private final ChangeUpdate.Factory updateFactory;
private final CommentsUtil commentsUtil;
private final NoteDbUpdateManager.Factory updateManagerFactory;
private final NotesMigration migration;
private final PatchListCache patchListCache;
private final PersonIdent serverIdent;
private final ProjectCache projectCache;
private final String serverId;
private final long skewMs;
@Inject
ChangeRebuilderImpl(
@GerritServerConfig Config cfg,
SchemaFactory<ReviewDb> schemaFactory,
ChangeBundleReader bundleReader,
ChangeDraftUpdate.Factory draftUpdateFactory,
ChangeNoteUtil changeNoteUtil,
ChangeNotes.Factory notesFactory,
ChangeUpdate.Factory updateFactory,
CommentsUtil commentsUtil,
NoteDbUpdateManager.Factory updateManagerFactory,
NotesMigration migration,
PatchListCache patchListCache,
@GerritPersonIdent PersonIdent serverIdent,
@Nullable ProjectCache projectCache,
@GerritServerId String serverId) {
super(schemaFactory);
this.bundleReader = bundleReader;
this.draftUpdateFactory = draftUpdateFactory;
this.changeNoteUtil = changeNoteUtil;
this.notesFactory = notesFactory;
this.updateFactory = updateFactory;
this.commentsUtil = commentsUtil;
this.updateManagerFactory = updateManagerFactory;
this.migration = migration;
this.patchListCache = patchListCache;
this.serverIdent = serverIdent;
this.projectCache = projectCache;
this.serverId = serverId;
this.skewMs = NoteDbChangeState.getReadOnlySkew(cfg);
}
@Override
public Result rebuild(ReviewDb db, Change.Id changeId) throws IOException, OrmException {
return rebuild(db, changeId, true);
}
@Override
public Result rebuildEvenIfReadOnly(ReviewDb db, Change.Id changeId)
throws IOException, OrmException {
return rebuild(db, changeId, false);
}
private Result rebuild(ReviewDb db, Change.Id changeId, boolean checkReadOnly)
throws IOException, OrmException {
db = ReviewDbUtil.unwrapDb(db);
// Read change just to get project; this instance is then discarded so we can read a consistent
// ChangeBundle inside a transaction.
Change change = db.changes().get(changeId);
if (change == null) {
throw new NoSuchChangeException(changeId);
}
try (NoteDbUpdateManager manager = updateManagerFactory.create(change.getProject())) {
buildUpdates(manager, bundleReader.fromReviewDb(db, changeId));
return execute(db, changeId, manager, checkReadOnly, true);
}
}
@Override
public Result rebuild(NoteDbUpdateManager manager, ChangeBundle bundle)
throws NoSuchChangeException, IOException, OrmException {
Change change = new Change(bundle.getChange());
buildUpdates(manager, bundle);
return manager.stageAndApplyDelta(change);
}
@Override
public NoteDbUpdateManager stage(ReviewDb db, Change.Id changeId)
throws IOException, OrmException {
db = ReviewDbUtil.unwrapDb(db);
Change change = checkNoteDbState(ChangeNotes.readOneReviewDbChange(db, changeId));
if (change == null) {
throw new NoSuchChangeException(changeId);
}
NoteDbUpdateManager manager = updateManagerFactory.create(change.getProject());
buildUpdates(manager, bundleReader.fromReviewDb(db, changeId));
manager.stage();
return manager;
}
@Override
public Result execute(ReviewDb db, Change.Id changeId, NoteDbUpdateManager manager)
throws OrmException, IOException {
return execute(db, changeId, manager, true, true);
}
public Result execute(
ReviewDb db,
Change.Id changeId,
NoteDbUpdateManager manager,
boolean checkReadOnly,
boolean executeManager)
throws OrmException, IOException {
db = ReviewDbUtil.unwrapDb(db);
Change change = checkNoteDbState(ChangeNotes.readOneReviewDbChange(db, changeId));
if (change == null) {
throw new NoSuchChangeException(changeId);
}
String oldNoteDbStateStr = change.getNoteDbState();
Result r = manager.stageAndApplyDelta(change);
String newNoteDbStateStr = change.getNoteDbState();
if (newNoteDbStateStr == null) {
throw new OrmException(
String.format(
"Rebuilding change %s produced no writes to NoteDb: %s",
changeId, bundleReader.fromReviewDb(db, changeId)));
}
NoteDbChangeState newNoteDbState =
requireNonNull(NoteDbChangeState.parse(changeId, newNoteDbStateStr));
try {
db.changes()
.atomicUpdate(
changeId,
new AtomicUpdate<Change>() {
@Override
public Change update(Change change) {
if (checkReadOnly) {
NoteDbChangeState.checkNotReadOnly(change, skewMs);
}
String currNoteDbStateStr = change.getNoteDbState();
if (Objects.equals(currNoteDbStateStr, newNoteDbStateStr)) {
// Another thread completed the same rebuild we were about to.
throw new AbortUpdateException();
} else if (!Objects.equals(oldNoteDbStateStr, currNoteDbStateStr)) {
// Another thread updated the state to something else.
throw new ConflictingUpdateRuntimeException(change, oldNoteDbStateStr);
}
change.setNoteDbState(newNoteDbStateStr);
return change;
}
});
} catch (ConflictingUpdateRuntimeException e) {
// Rethrow as an OrmException so the caller knows to use staged results. Strictly speaking
// they are not completely up to date, but result we send to the caller is the same as if this
// rebuild had executed before the other thread.
throw new ConflictingUpdateException(e);
} catch (AbortUpdateException e) {
if (newNoteDbState.isUpToDate(
manager.getChangeRepo().cmds.getRepoRefCache(),
manager.getAllUsersRepo().cmds.getRepoRefCache())) {
// If the state in ReviewDb matches NoteDb at this point, it means another thread
// successfully completed this rebuild. It's ok to not execute the update in this case,
// since the object referenced in the Result was flushed to the repo by whatever thread won
// the race.
return r;
}
// If the state doesn't match, that means another thread attempted this rebuild, but
// failed. Fall through and try to update the ref again.
}
if (migration.failChangeWrites()) {
// Don't even attempt to execute if read-only, it would fail anyway. But do throw an exception
// to the caller so they know to use the staged results instead of reading from the repo.
throw new OrmException(NoteDbUpdateManager.CHANGES_READ_ONLY);
}
if (executeManager) {
manager.execute();
}
return r;
}
static Change checkNoteDbState(Change c) throws OrmException {
// Can only rebuild a change if its primary storage is ReviewDb.
NoteDbChangeState s = NoteDbChangeState.parse(c);
if (s != null && s.getPrimaryStorage() != PrimaryStorage.REVIEW_DB) {
throw new OrmException(String.format("cannot rebuild change %s with state %s", c.getId(), s));
}
return c;
}
@Override
public void buildUpdates(NoteDbUpdateManager manager, ChangeBundle bundle)
throws IOException, OrmException {
manager.setCheckExpectedState(false).setRefLogMessage("Rebuilding change");
Change change = new Change(bundle.getChange());
if (bundle.getPatchSets().isEmpty()) {
throw new NoPatchSetsException(change.getId());
}
if (change.getLastUpdatedOn().compareTo(change.getCreatedOn()) < 0) {
// A bug in data migration might set created_on to the time of the migration. The
// correct timestamps were lost, but we can at least set it so created_on is not after
// last_updated_on.
// See https://bugs.chromium.org/p/gerrit/issues/detail?id=7397
change.setCreatedOn(change.getLastUpdatedOn());
}
// We will rebuild all events, except for draft comments, in buckets based on author and
// timestamp.
List<Event> events = new ArrayList<>();
ListMultimap<Account.Id, DraftCommentEvent> draftCommentEvents =
MultimapBuilder.hashKeys().arrayListValues().build();
events.addAll(getHashtagsEvents(change, manager));
// Delete ref only after hashtags have been read.
deleteChangeMetaRef(change, manager.getChangeRepo().cmds);
deleteDraftRefs(change, manager.getAllUsersRepo());
Integer minPsNum = getMinPatchSetNum(bundle);
TreeMap<PatchSet.Id, PatchSetEvent> patchSetEvents =
new TreeMap<>(ReviewDbUtil.intKeyOrdering());
for (PatchSet ps : bundle.getPatchSets()) {
PatchSetEvent pse = new PatchSetEvent(change, ps, manager.getChangeRepo().rw);
patchSetEvents.put(ps.getId(), pse);
events.add(pse);
for (Comment c : getComments(bundle, serverId, Status.PUBLISHED, ps)) {
CommentEvent e = new CommentEvent(c, change, ps, patchListCache);
events.add(e.addDep(pse));
}
for (Comment c : getComments(bundle, serverId, Status.DRAFT, ps)) {
DraftCommentEvent e = new DraftCommentEvent(c, change, ps, patchListCache);
draftCommentEvents.put(c.author.getId(), e);
}
}
ensurePatchSetOrder(patchSetEvents);
for (PatchSetApproval psa : bundle.getPatchSetApprovals()) {
PatchSetEvent pse = patchSetEvents.get(psa.getPatchSetId());
if (pse != null) {
events.add(new ApprovalEvent(psa, change.getCreatedOn()).addDep(pse));
}
}
for (Table.Cell<ReviewerStateInternal, Account.Id, Timestamp> r :
bundle.getReviewers().asTable().cellSet()) {
events.add(new ReviewerEvent(r, change.getCreatedOn()));
}
Change noteDbChange = new Change(null, null, null, null, null);
for (ChangeMessage msg : bundle.getChangeMessages()) {
Event msgEvent = new ChangeMessageEvent(change, noteDbChange, msg, change.getCreatedOn());
if (msg.getPatchSetId() != null) {
PatchSetEvent pse = patchSetEvents.get(msg.getPatchSetId());
if (pse == null) {
continue; // Ignore events for missing patch sets.
}
msgEvent.addDep(pse);
}
events.add(msgEvent);
}
sortAndFillEvents(change, noteDbChange, bundle.getPatchSets(), events, minPsNum);
EventList<Event> el = new EventList<>();
for (Event e : events) {
if (!el.canAdd(e)) {
flushEventsToUpdate(manager, el, change);
checkState(el.canAdd(e));
}
el.add(e);
}
flushEventsToUpdate(manager, el, change);
EventList<DraftCommentEvent> plcel = new EventList<>();
for (Account.Id author : draftCommentEvents.keys()) {
for (DraftCommentEvent e : Ordering.natural().sortedCopy(draftCommentEvents.get(author))) {
if (!plcel.canAdd(e)) {
flushEventsToDraftUpdate(manager, plcel, change);
checkState(plcel.canAdd(e));
}
plcel.add(e);
}
flushEventsToDraftUpdate(manager, plcel, change);
}
}
private static Integer getMinPatchSetNum(ChangeBundle bundle) {
Integer minPsNum = null;
for (PatchSet ps : bundle.getPatchSets()) {
int n = ps.getId().get();
if (minPsNum == null || n < minPsNum) {
minPsNum = n;
}
}
return minPsNum;
}
private static void ensurePatchSetOrder(TreeMap<PatchSet.Id, PatchSetEvent> events) {
if (events.isEmpty()) {
return;
}
Iterator<PatchSetEvent> it = events.values().iterator();
PatchSetEvent curr = it.next();
while (it.hasNext()) {
PatchSetEvent next = it.next();
next.addDep(curr);
curr = next;
}
}
private static List<Comment> getComments(
ChangeBundle bundle, String serverId, PatchLineComment.Status status, PatchSet ps) {
return bundle
.getPatchLineComments()
.stream()
.filter(c -> c.getPatchSetId().equals(ps.getId()) && c.getStatus() == status)
.map(plc -> plc.asComment(serverId))
.sorted(CommentsUtil.COMMENT_ORDER)
.collect(toList());
}
private void sortAndFillEvents(
Change change,
Change noteDbChange,
ImmutableCollection<PatchSet> patchSets,
List<Event> events,
Integer minPsNum) {
Event finalUpdates = new FinalUpdatesEvent(change, noteDbChange, patchSets);
events.add(finalUpdates);
setPostSubmitDeps(events);
new EventSorter(events).sort();
// Ensure the first event in the list creates the change, setting the author and any required
// footers. Also force the creation time of the first patch set to match the creation time of
// the change.
Event first = events.get(0);
if (first instanceof PatchSetEvent && change.getOwner().equals(first.user)) {
first.when = change.getCreatedOn();
((PatchSetEvent) first).createChange = true;
} else {
events.add(0, new CreateChangeEvent(change, minPsNum));
}
// Final pass to correct some inconsistencies.
//
// First, fill in any missing patch set IDs using the latest patch set of the change at the time
// of the event, because NoteDb can't represent actions with no associated patch set ID. This
// workaround is as if a user added a ChangeMessage on the change by replying from the latest
// patch set.
//
// Start with the first patch set that actually exists. If there are no patch sets at all,
// minPsNum will be null, so just bail and use 1 as the patch set ID.
//
// Second, ensure timestamps are nondecreasing, by copying the previous timestamp if this
// happens. This assumes that the only way this can happen is due to dependency constraints, and
// it is ok to give an event the same timestamp as one of its dependencies.
int ps = firstNonNull(minPsNum, 1);
for (int i = 0; i < events.size(); i++) {
Event e = events.get(i);
if (e.psId == null) {
e.psId = new PatchSet.Id(change.getId(), ps);
} else {
ps = Math.max(ps, e.psId.get());
}
if (i > 0) {
Event p = events.get(i - 1);
if (e.when.before(p.when)) {
e.when = p.when;
}
}
}
}
private void setPostSubmitDeps(List<Event> events) {
Optional<Event> submitEvent =
Lists.reverse(events).stream().filter(Event::isSubmit).findFirst();
if (submitEvent.isPresent()) {
events.stream().filter(Event::isPostSubmitApproval).forEach(e -> e.addDep(submitEvent.get()));
}
}
private void flushEventsToUpdate(
NoteDbUpdateManager manager, EventList<Event> events, Change change)
throws OrmException, IOException {
if (events.isEmpty()) {
return;
}
Comparator<String> labelNameComparator;
if (projectCache != null) {
labelNameComparator = projectCache.get(change.getProject()).getLabelTypes().nameComparator();
} else {
// No project cache available, bail and use natural ordering; there's no semantic difference
// anyway difference.
labelNameComparator = Ordering.natural();
}
ChangeUpdate update =
updateFactory.create(
change,
events.getAccountId(),
events.getRealAccountId(),
newAuthorIdent(events),
events.getWhen(),
labelNameComparator);
update.setAllowWriteToNewRef(true);
update.setPatchSetId(events.getPatchSetId());
update.setTag(events.getTag());
for (Event e : events) {
e.apply(update);
}
manager.add(update);
events.clear();
}
private void flushEventsToDraftUpdate(
NoteDbUpdateManager manager, EventList<DraftCommentEvent> events, Change change) {
if (events.isEmpty()) {
return;
}
ChangeDraftUpdate update =
draftUpdateFactory.create(
change,
events.getAccountId(),
events.getRealAccountId(),
newAuthorIdent(events),
events.getWhen());
update.setPatchSetId(events.getPatchSetId());
for (DraftCommentEvent e : events) {
e.applyDraft(update);
}
manager.add(update);
events.clear();
}
private PersonIdent newAuthorIdent(EventList<?> events) {
Account.Id id = events.getAccountId();
if (id == null) {
return new PersonIdent(serverIdent, events.getWhen());
}
return changeNoteUtil.newIdent(id, events.getWhen(), serverIdent);
}
private List<HashtagsEvent> getHashtagsEvents(Change change, NoteDbUpdateManager manager)
throws IOException {
String refName = changeMetaRef(change.getId());
Optional<ObjectId> old = manager.getChangeRepo().getObjectId(refName);
if (!old.isPresent()) {
return Collections.emptyList();
}
RevWalk rw = manager.getChangeRepo().rw;
List<HashtagsEvent> events = new ArrayList<>();
rw.reset();
rw.markStart(rw.parseCommit(old.get()));
for (RevCommit commit : rw) {
Account.Id authorId;
try {
authorId =
changeNoteUtil
.getLegacyChangeNoteRead()
.parseIdent(commit.getAuthorIdent(), change.getId());
} catch (ConfigInvalidException e) {
continue; // Corrupt data, no valid hashtags in this commit.
}
PatchSet.Id psId = parsePatchSetId(change, commit);
Set<String> hashtags = parseHashtags(commit);
if (authorId == null || psId == null || hashtags == null) {
continue;
}
Timestamp commitTime = new Timestamp(commit.getCommitterIdent().getWhen().getTime());
events.add(new HashtagsEvent(psId, authorId, commitTime, hashtags, change.getCreatedOn()));
}
return events;
}
private Set<String> parseHashtags(RevCommit commit) {
List<String> hashtagsLines = commit.getFooterLines(FOOTER_HASHTAGS);
if (hashtagsLines.isEmpty() || hashtagsLines.size() > 1) {
return null;
}
if (hashtagsLines.get(0).isEmpty()) {
return ImmutableSet.of();
}
return Sets.newHashSet(Splitter.on(',').split(hashtagsLines.get(0)));
}
private PatchSet.Id parsePatchSetId(Change change, RevCommit commit) {
List<String> psIdLines = commit.getFooterLines(FOOTER_PATCH_SET);
if (psIdLines.size() != 1) {
return null;
}
Integer psId = Ints.tryParse(psIdLines.get(0));
if (psId == null) {
return null;
}
return new PatchSet.Id(change.getId(), psId);
}
private void deleteChangeMetaRef(Change change, ChainedReceiveCommands cmds) throws IOException {
String refName = changeMetaRef(change.getId());
Optional<ObjectId> old = cmds.get(refName);
if (old.isPresent()) {
cmds.add(new ReceiveCommand(old.get(), ObjectId.zeroId(), refName));
}
}
private void deleteDraftRefs(Change change, OpenRepo allUsersRepo) throws IOException {
for (Ref r :
allUsersRepo
.repo
.getRefDatabase()
.getRefsByPrefix(RefNames.refsDraftCommentsPrefix(change.getId()))) {
allUsersRepo.cmds.add(new ReceiveCommand(r.getObjectId(), ObjectId.zeroId(), r.getName()));
}
}
static void createChange(ChangeUpdate update, Change change) {
update.setSubjectForCommit("Create change");
update.setChangeId(change.getKey().get());
update.setBranch(change.getDest().get());
update.setSubject(change.getOriginalSubject());
if (change.getRevertOf() != null) {
update.setRevertOf(change.getRevertOf().get());
}
}
@Override
public void rebuildReviewDb(ReviewDb db, Project.NameKey project, Change.Id changeId)
throws OrmException {
// TODO(dborowitz): Fail fast if changes tables are disabled in ReviewDb.
ChangeNotes notes = notesFactory.create(db, project, changeId);
ChangeBundle bundle = ChangeBundle.fromNotes(commentsUtil, notes);
db = ReviewDbUtil.unwrapDb(db);
db.changes().beginTransaction(changeId);
try {
Change c = db.changes().get(changeId);
if (c != null) {
PrimaryStorage ps = PrimaryStorage.of(c);
switch (ps) {
case REVIEW_DB:
return; // Nothing to do.
case NOTE_DB:
break; // Continue and rebuild.
default:
throw new OrmException("primary storage of " + changeId + " is " + ps);
}
} else {
c = notes.getChange();
}
db.changes().upsert(Collections.singleton(c));
putExactlyEntities(
db.changeMessages(), db.changeMessages().byChange(c.getId()), bundle.getChangeMessages());
putExactlyEntities(db.patchSets(), db.patchSets().byChange(c.getId()), bundle.getPatchSets());
putExactlyEntities(
db.patchSetApprovals(),
db.patchSetApprovals().byChange(c.getId()),
bundle.getPatchSetApprovals());
putExactlyEntities(
db.patchComments(),
db.patchComments().byChange(c.getId()),
bundle.getPatchLineComments());
db.commit();
} finally {
db.rollback();
}
}
private static <T, K extends Key<?>> void putExactlyEntities(
Access<T, K> access, Iterable<T> existing, Collection<T> ents) throws OrmException {
Set<K> toKeep = access.toMap(ents).keySet();
access.delete(
FluentIterable.from(existing).filter(e -> !toKeep.contains(access.primaryKey(e))));
access.upsert(ents);
}
}

82
java/com/google/gerrit/server/notedb/rebuild/CommentEvent.java
View File

@@ -1,82 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import static com.google.gerrit.server.CommentsUtil.setCommentRevId;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Comment;
import com.google.gerrit.reviewdb.client.PatchLineComment;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.server.CommentsUtil;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gerrit.server.patch.PatchListCache;
import com.google.gerrit.server.patch.PatchListNotAvailableException;
class CommentEvent extends Event {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
public final Comment c;
private final Change change;
private final PatchSet ps;
private final PatchListCache cache;
CommentEvent(Comment c, Change change, PatchSet ps, PatchListCache cache) {
super(
CommentsUtil.getCommentPsId(change.getId(), c),
c.author.getId(),
c.getRealAuthor().getId(),
c.writtenOn,
change.getCreatedOn(),
c.tag);
this.c = c;
this.change = change;
this.ps = ps;
this.cache = cache;
}
@Override
boolean uniquePerUpdate() {
return false;
}
@Override
protected boolean canHaveTag() {
return true;
}
@Override
void apply(ChangeUpdate update) {
checkUpdate(update);
if (c.revId == null) {
try {
setCommentRevId(c, cache, change, ps);
} catch (PatchListNotAvailableException e) {
logger.atWarning().log(
"Unable to determine parent commit of patch set %s (%s); omitting inline comment %s",
ps.getId(), ps.getRevision(), c);
return;
}
}
update.putComment(PatchLineComment.Status.PUBLISHED, c);
}
@Override
protected void addToString(ToStringHelper helper) {
helper.add("message", c.message);
}
}

32
java/com/google/gerrit/server/notedb/rebuild/ConflictingUpdateException.java
View File

@@ -1,32 +0,0 @@
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.gwtorm.server.OrmException;
/**
* {@link com.google.gwtorm.server.OrmException} thrown by {@link ChangeRebuilder} when rebuilding a
* change failed because another operation modified its {@link
* com.google.gerrit.server.notedb.NoteDbChangeState}.
*/
public class ConflictingUpdateException extends OrmException {
private static final long serialVersionUID = 1L;
// Always created from a ConflictingUpdateRuntimeException because it originates from an
// AtomicUpdate, which cannot throw checked exceptions.
ConflictingUpdateException(ConflictingUpdateRuntimeException cause) {
super(cause.getMessage(), cause);
}
}

29
java/com/google/gerrit/server/notedb/rebuild/ConflictingUpdateRuntimeException.java
View File

@@ -1,29 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gwtorm.server.OrmRuntimeException;
class ConflictingUpdateRuntimeException extends OrmRuntimeException {
private static final long serialVersionUID = 1L;
ConflictingUpdateRuntimeException(Change change, String expectedNoteDbState) {
super(
String.format(
"Expected change %s to have noteDbState %s but was %s",
change.getId(), expectedNoteDbState, change.getNoteDbState()));
}
}

59
java/com/google/gerrit/server/notedb/rebuild/CreateChangeEvent.java
View File

@@ -1,59 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gwtorm.server.OrmException;
import java.io.IOException;
class CreateChangeEvent extends Event {
private final Change change;
private static PatchSet.Id psId(Change change, Integer minPsNum) {
int n;
if (minPsNum == null) {
// There were no patch sets for the change at all, so something is very
// wrong. Bail and use 1 as the patch set.
n = 1;
} else {
n = minPsNum;
}
return new PatchSet.Id(change.getId(), n);
}
CreateChangeEvent(Change change, Integer minPsNum) {
super(
psId(change, minPsNum),
change.getOwner(),
change.getOwner(),
change.getCreatedOn(),
change.getCreatedOn(),
null);
this.change = change;
}
@Override
boolean uniquePerUpdate() {
return true;
}
@Override
void apply(ChangeUpdate update) throws IOException, OrmException {
checkUpdate(update);
ChangeRebuilderImpl.createChange(update, change);
}
}

81
java/com/google/gerrit/server/notedb/rebuild/DraftCommentEvent.java
View File

@@ -1,81 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import static com.google.gerrit.server.CommentsUtil.setCommentRevId;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Comment;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.server.CommentsUtil;
import com.google.gerrit.server.notedb.ChangeDraftUpdate;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gerrit.server.patch.PatchListCache;
import com.google.gerrit.server.patch.PatchListNotAvailableException;
class DraftCommentEvent extends Event {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
public final Comment c;
private final Change change;
private final PatchSet ps;
private final PatchListCache cache;
DraftCommentEvent(Comment c, Change change, PatchSet ps, PatchListCache cache) {
super(
CommentsUtil.getCommentPsId(change.getId(), c),
c.author.getId(),
c.getRealAuthor().getId(),
c.writtenOn,
change.getCreatedOn(),
c.tag);
this.c = c;
this.change = change;
this.ps = ps;
this.cache = cache;
}
@Override
boolean uniquePerUpdate() {
return false;
}
@Override
void apply(ChangeUpdate update) {
throw new UnsupportedOperationException();
}
void applyDraft(ChangeDraftUpdate draftUpdate) {
if (c.revId == null) {
try {
setCommentRevId(c, cache, change, ps);
} catch (PatchListNotAvailableException e) {
logger.atWarning().log(
"Unable to determine parent commit of patch set %s (%s);"
+ " omitting draft inline comment %s",
ps.getId(), ps.getRevision(), c);
return;
}
}
draftUpdate.putComment(c);
}
@Override
protected void addToString(ToStringHelper helper) {
helper.add("message", c.message);
}
}

146
java/com/google/gerrit/server/notedb/rebuild/Event.java
View File

@@ -1,146 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import static com.google.common.base.Preconditions.checkState;
import static com.google.gerrit.server.notedb.rebuild.ChangeRebuilderImpl.MAX_WINDOW_MS;
import com.google.common.base.MoreObjects;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.common.collect.ComparisonChain;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.server.ReviewDbUtil;
import com.google.gerrit.server.notedb.AbstractChangeUpdate;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gwtorm.server.OrmException;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
abstract class Event implements Comparable<Event> {
// NOTE: EventList only supports direct subclasses, not an arbitrary
// hierarchy.
final Account.Id user;
final Account.Id realUser;
final String tag;
final boolean predatesChange;
/** Dependencies of this event; other events that must happen before this one. */
final List<Event> deps;
Timestamp when;
PatchSet.Id psId;
protected Event(
PatchSet.Id psId,
Account.Id effectiveUser,
Account.Id realUser,
Timestamp when,
Timestamp changeCreatedOn,
String tag) {
this.psId = psId;
this.user = effectiveUser;
this.realUser = realUser != null ? realUser : effectiveUser;
this.tag = tag;
// Truncate timestamps at the change's createdOn timestamp.
predatesChange = when.before(changeCreatedOn);
this.when = predatesChange ? changeCreatedOn : when;
deps = new ArrayList<>();
}
protected void checkUpdate(AbstractChangeUpdate update) {
checkState(
Objects.equals(update.getPatchSetId(), psId),
"cannot apply event for %s to update for %s",
update.getPatchSetId(),
psId);
checkState(
when.getTime() - update.getWhen().getTime() <= MAX_WINDOW_MS,
"event at %s outside update window starting at %s",
when,
update.getWhen());
checkState(
Objects.equals(update.getNullableAccountId(), user),
"cannot apply event by %s to update by %s",
user,
update.getNullableAccountId());
}
Event addDep(Event e) {
deps.add(e);
return this;
}
/**
* @return whether this event type must be unique per {@link ChangeUpdate}, i.e. there may be at
* most one of this type.
*/
abstract boolean uniquePerUpdate();
abstract void apply(ChangeUpdate update) throws OrmException, IOException;
protected boolean isPostSubmitApproval() {
return false;
}
protected boolean isSubmit() {
return false;
}
protected boolean canHaveTag() {
return false;
}
@Override
public String toString() {
ToStringHelper helper =
MoreObjects.toStringHelper(this)
.add("psId", psId)
.add("effectiveUser", user)
.add("realUser", realUser)
.add("when", when)
.add("tag", tag);
addToString(helper);
return helper.toString();
}
/** @param helper toString helper to add fields to */
protected void addToString(ToStringHelper helper) {}
@Override
public int compareTo(Event other) {
return ComparisonChain.start()
.compareFalseFirst(this.isFinalUpdates(), other.isFinalUpdates())
.compare(this.when, other.when)
.compareTrueFirst(isPatchSet(), isPatchSet())
.compareTrueFirst(this.predatesChange, other.predatesChange)
.compare(this.user, other.user, ReviewDbUtil.intKeyOrdering())
.compare(this.realUser, other.realUser, ReviewDbUtil.intKeyOrdering())
.compare(this.psId, other.psId, ReviewDbUtil.intKeyOrdering().nullsLast())
.result();
}
private boolean isPatchSet() {
return this instanceof PatchSetEvent;
}
private boolean isFinalUpdates() {
return this instanceof FinalUpdatesEvent;
}
}

170
java/com/google/gerrit/server/notedb/rebuild/EventList.java
View File

@@ -1,170 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.Lists;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.PatchSet;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Objects;
class EventList<E extends Event> implements Iterable<E> {
private final ArrayList<E> list = new ArrayList<>();
private boolean isSubmit;
@Override
public Iterator<E> iterator() {
return list.iterator();
}
void add(E e) {
list.add(e);
if (e.isSubmit()) {
isSubmit = true;
}
}
void clear() {
list.clear();
isSubmit = false;
}
boolean isEmpty() {
return list.isEmpty();
}
boolean canAdd(E e) {
if (isEmpty()) {
return true;
}
if (e instanceof FinalUpdatesEvent) {
return false; // FinalUpdatesEvent always gets its own update.
}
Event last = getLast();
if (!Objects.equals(e.user, last.user)
|| !Objects.equals(e.realUser, last.realUser)
|| !e.psId.equals(last.psId)) {
return false; // Different patch set or author.
}
if (e.canHaveTag() && canHaveTag() && !Objects.equals(e.tag, getTag())) {
// We should trust the tag field, and it doesn't match.
return false;
}
if (e.isPostSubmitApproval() && isSubmit) {
// Post-submit approvals must come after the update that submits.
return false;
}
long t = e.when.getTime();
long tFirst = getFirstTime();
long tLast = getLastTime();
checkArgument(t >= tLast, "event %s is before previous event in list %s", e, last);
if (t - tLast > ChangeRebuilderImpl.MAX_DELTA_MS
|| t - tFirst > ChangeRebuilderImpl.MAX_WINDOW_MS) {
return false; // Too much time elapsed.
}
if (!e.uniquePerUpdate()) {
return true;
}
for (Event o : this) {
if (e.getClass() == o.getClass()) {
return false; // Only one event of this type allowed per update.
}
}
// TODO(dborowitz): Additional heuristics, like keeping events separate if
// they affect overlapping fields within a single entity.
return true;
}
Timestamp getWhen() {
return get(0).when;
}
PatchSet.Id getPatchSetId() {
PatchSet.Id id = requireNonNull(get(0).psId);
for (int i = 1; i < size(); i++) {
checkState(
get(i).psId.equals(id), "mismatched patch sets in EventList: %s != %s", id, get(i).psId);
}
return id;
}
Account.Id getAccountId() {
Account.Id id = get(0).user;
for (int i = 1; i < size(); i++) {
checkState(
Objects.equals(id, get(i).user),
"mismatched users in EventList: %s != %s",
id,
get(i).user);
}
return id;
}
Account.Id getRealAccountId() {
Account.Id id = get(0).realUser;
for (int i = 1; i < size(); i++) {
checkState(
Objects.equals(id, get(i).realUser),
"mismatched real users in EventList: %s != %s",
id,
get(i).realUser);
}
return id;
}
String getTag() {
for (E e : Lists.reverse(list)) {
if (e.tag != null) {
return e.tag;
}
}
return null;
}
private boolean canHaveTag() {
return list.stream().anyMatch(Event::canHaveTag);
}
private E get(int i) {
return list.get(i);
}
private int size() {
return list.size();
}
private E getLast() {
return list.get(list.size() - 1);
}
private long getLastTime() {
return getLast().when.getTime();
}
private long getFirstTime() {
return list.get(0).when.getTime();
}
}

112
java/com/google/gerrit/server/notedb/rebuild/EventSorter.java
View File

@@ -1,112 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.SetMultimap;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.PriorityQueue;
/**
* Helper to sort a list of events.
*
* <p>Events are sorted in two passes:
*
* <ol>
* <li>Sort by natural order (timestamp, patch set, author, etc.)
* <li>Postpone any events with dependencies to occur only after all of their dependencies, where
* this violates natural order.
* </ol>
*
* {@link #sort()} modifies the event list in place (similar to {@link Collections#sort(List)}), but
* does not modify any event. In particular, events might end up out of order with respect to
* timestamp; callers are responsible for adjusting timestamps later if they prefer monotonicity.
*/
class EventSorter {
private final List<Event> out;
private final LinkedHashSet<Event> sorted;
private ListMultimap<Event, Event> waiting;
private SetMultimap<Event, Event> deps;
EventSorter(List<Event> events) {
LinkedHashSet<Event> all = new LinkedHashSet<>(events);
out = events;
for (Event e : events) {
for (Event d : e.deps) {
checkArgument(all.contains(d), "dep %s of %s not in input list", d, e);
}
}
all.clear();
sorted = all; // Presized.
}
void sort() {
// First pass: sort by natural order.
PriorityQueue<Event> todo = new PriorityQueue<>(out);
// Populate waiting map after initial sort to preserve natural order.
waiting = MultimapBuilder.hashKeys().arrayListValues().build();
deps = MultimapBuilder.hashKeys().hashSetValues().build();
for (Event e : todo) {
for (Event d : e.deps) {
deps.put(e, d);
waiting.put(d, e);
}
}
// Second pass: enforce dependencies.
int size = out.size();
while (!todo.isEmpty()) {
process(todo.remove(), todo);
}
checkState(
sorted.size() == size, "event sort expected %s elements, got %s", size, sorted.size());
// Modify out in-place a la Collections#sort.
out.clear();
out.addAll(sorted);
}
void process(Event e, PriorityQueue<Event> todo) {
if (sorted.contains(e)) {
return; // Already emitted.
}
if (!deps.get(e).isEmpty()) {
// Not all events that e depends on have been emitted yet. Ignore e for
// now; it will get added back to the queue in the block below once its
// last dependency is processed.
return;
}
// All events that e depends on have been emitted, so e can be emitted.
sorted.add(e);
// Remove e from the dependency set of all events waiting on e, and add
// those events back to the queue in the original priority order for
// reconsideration.
for (Event w : waiting.get(e)) {
deps.get(w).remove(e);
todo.add(w);
}
}
}

101
java/com/google/gerrit/server/notedb/rebuild/FinalUpdatesEvent.java
View File

@@ -1,101 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import static com.google.gerrit.reviewdb.server.ReviewDbUtil.intKeyOrdering;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.common.collect.ImmutableCollection;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gwtorm.server.OrmException;
import java.util.Objects;
class FinalUpdatesEvent extends Event {
private final Change change;
private final Change noteDbChange;
private final ImmutableCollection<PatchSet> patchSets;
FinalUpdatesEvent(Change change, Change noteDbChange, ImmutableCollection<PatchSet> patchSets) {
super(
change.currentPatchSetId(),
change.getOwner(),
change.getOwner(),
change.getLastUpdatedOn(),
change.getCreatedOn(),
null);
this.change = change;
this.noteDbChange = noteDbChange;
this.patchSets = patchSets;
}
@Override
boolean uniquePerUpdate() {
return true;
}
@SuppressWarnings("deprecation")
@Override
void apply(ChangeUpdate update) throws OrmException {
if (!Objects.equals(change.getTopic(), noteDbChange.getTopic())) {
update.setTopic(change.getTopic());
}
if (!statusMatches()) {
// TODO(dborowitz): Stamp approximate approvals at this time.
update.fixStatus(change.getStatus());
}
if (change.isPrivate() != noteDbChange.isPrivate()) {
update.setPrivate(change.isPrivate());
}
if (change.isWorkInProgress() != noteDbChange.isWorkInProgress()) {
update.setWorkInProgress(change.isWorkInProgress());
}
if (change.getSubmissionId() != null && noteDbChange.getSubmissionId() == null) {
update.setSubmissionId(change.getSubmissionId());
}
if (!Objects.equals(change.getAssignee(), noteDbChange.getAssignee())) {
// TODO(dborowitz): Parse intermediate values out from messages.
update.setAssignee(change.getAssignee());
}
if (!patchSets.isEmpty() && !highestNumberedPatchSetIsCurrent()) {
update.setCurrentPatchSet();
}
if (!update.isEmpty()) {
update.setSubjectForCommit("Final NoteDb migration updates");
}
}
private boolean statusMatches() {
return Objects.equals(change.getStatus(), noteDbChange.getStatus());
}
private boolean highestNumberedPatchSetIsCurrent() {
PatchSet.Id max = patchSets.stream().map(PatchSet::getId).max(intKeyOrdering()).get();
return max.equals(change.currentPatchSetId());
}
@Override
protected boolean isSubmit() {
return change.getStatus() == Change.Status.MERGED;
}
@Override
protected void addToString(ToStringHelper helper) {
if (!statusMatches()) {
helper.add("status", change.getStatus());
}
}
}

121
java/com/google/gerrit/server/notedb/rebuild/GcAllUsers.java
View File

@@ -1,121 +0,0 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import static java.util.Objects.requireNonNull;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_GC_SECTION;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_AUTO;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.common.data.GarbageCollectionResult;
import com.google.gerrit.server.config.AllUsersName;
import com.google.gerrit.server.git.GarbageCollection;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.LocalDiskRepositoryManager;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.function.Consumer;
import org.eclipse.jgit.lib.Repository;
@Singleton
public class GcAllUsers {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final AllUsersName allUsers;
private final GarbageCollection.Factory gcFactory;
private final GitRepositoryManager repoManager;
@Inject
GcAllUsers(
AllUsersName allUsers,
GarbageCollection.Factory gcFactory,
GitRepositoryManager repoManager) {
this.allUsers = allUsers;
this.gcFactory = gcFactory;
this.repoManager = repoManager;
}
public void runWithLogger() {
// Print log messages using logger, and skip progress.
run(s -> logger.atInfo().log(s), null);
}
public void run(PrintWriter writer) {
// Print both log messages and progress to given writer.
run(requireNonNull(writer)::println, writer);
}
private void run(Consumer<String> logOneLine, @Nullable PrintWriter progressWriter) {
if (!(repoManager instanceof LocalDiskRepositoryManager)) {
logOneLine.accept("Skipping GC of " + allUsers + "; not a local disk repo");
return;
}
if (!enableAutoGc(logOneLine)) {
logOneLine.accept(
"Skipping GC of "
+ allUsers
+ " due to disabling "
+ CONFIG_GC_SECTION
+ "."
+ CONFIG_KEY_AUTO);
logOneLine.accept(
"If loading accounts is slow after the NoteDb migration, run `git gc` on "
+ allUsers
+ " manually");
return;
}
if (progressWriter == null) {
// Mimic log line from GarbageCollection.
logOneLine.accept("collecting garbage for \"" + allUsers + "\":\n");
}
GarbageCollectionResult result =
gcFactory.create().run(ImmutableList.of(allUsers), progressWriter);
if (!result.hasErrors()) {
return;
}
for (GarbageCollectionResult.Error e : result.getErrors()) {
switch (e.getType()) {
case GC_ALREADY_SCHEDULED:
logOneLine.accept("GC already scheduled for " + e.getProjectName());
break;
case GC_FAILED:
logOneLine.accept("GC failed for " + e.getProjectName());
break;
case REPOSITORY_NOT_FOUND:
logOneLine.accept(e.getProjectName() + " repo not found");
break;
default:
logOneLine.accept("GC failed for " + e.getProjectName() + ": " + e.getType());
break;
}
}
}
private boolean enableAutoGc(Consumer<String> logOneLine) {
try (Repository repo = repoManager.openRepository(allUsers)) {
return repo.getConfig().getInt(CONFIG_GC_SECTION, CONFIG_KEY_AUTO, -1) != 0;
} catch (IOException e) {
logOneLine.accept(
"Error reading config for " + allUsers + ":\n" + Throwables.getStackTraceAsString(e));
return false;
}
}
}

62
java/com/google/gerrit/server/notedb/rebuild/HashtagsEvent.java
View File

@@ -1,62 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gwtorm.server.OrmException;
import java.sql.Timestamp;
import java.util.Set;
class HashtagsEvent extends Event {
private final Set<String> hashtags;
HashtagsEvent(
PatchSet.Id psId,
Account.Id who,
Timestamp when,
Set<String> hashtags,
Timestamp changeCreatdOn) {
super(
psId,
who,
who,
when,
changeCreatdOn,
// Somewhat confusingly, hashtags do not use the setTag method on
// AbstractChangeUpdate, so pass null as the tag.
null);
this.hashtags = hashtags;
}
@Override
boolean uniquePerUpdate() {
// Since these are produced from existing commits in the old NoteDb graph,
// we know that there must be one per commit in the rebuilt graph.
return true;
}
@Override
void apply(ChangeUpdate update) throws OrmException {
update.setHashtags(hashtags);
}
@Override
protected void addToString(ToStringHelper helper) {
helper.add("hashtags", hashtags);
}
}

86
java/com/google/gerrit/server/notedb/rebuild/PatchSetEvent.java
View File

@@ -1,86 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gwtorm.server.OrmException;
import java.io.IOException;
import java.util.List;
import org.eclipse.jgit.errors.InvalidObjectIdException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.revwalk.RevWalk;
class PatchSetEvent extends Event {
private final Change change;
private final PatchSet ps;
private final RevWalk rw;
boolean createChange;
PatchSetEvent(Change change, PatchSet ps, RevWalk rw) {
super(
ps.getId(),
ps.getUploader(),
ps.getUploader(),
ps.getCreatedOn(),
change.getCreatedOn(),
null);
this.change = change;
this.ps = ps;
this.rw = rw;
}
@Override
boolean uniquePerUpdate() {
return true;
}
@Override
void apply(ChangeUpdate update) throws IOException, OrmException {
checkUpdate(update);
if (createChange) {
ChangeRebuilderImpl.createChange(update, change);
} else {
update.setSubject(change.getSubject());
update.setSubjectForCommit("Create patch set " + ps.getPatchSetId());
}
setRevision(update, ps);
update.setPsDescription(ps.getDescription());
List<String> groups = ps.getGroups();
if (!groups.isEmpty()) {
update.setGroups(ps.getGroups());
}
}
private void setRevision(ChangeUpdate update, PatchSet ps) throws IOException {
String rev = ps.getRevision().get();
String cert = ps.getPushCertificate();
ObjectId id;
try {
id = ObjectId.fromString(rev);
} catch (InvalidObjectIdException e) {
update.setRevisionForMissingCommit(rev, cert);
return;
}
try {
update.setCommit(rw, id, cert);
} catch (MissingObjectException e) {
update.setRevisionForMissingCommit(rev, cert);
return;
}
}
}

63
java/com/google/gerrit/server/notedb/rebuild/ReviewerEvent.java
View File

@@ -1,63 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.common.collect.Table;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gerrit.server.notedb.ReviewerStateInternal;
import com.google.gwtorm.server.OrmException;
import java.io.IOException;
import java.sql.Timestamp;
class ReviewerEvent extends Event {
private Table.Cell<ReviewerStateInternal, Account.Id, Timestamp> reviewer;
ReviewerEvent(
Table.Cell<ReviewerStateInternal, Account.Id, Timestamp> reviewer,
Timestamp changeCreatedOn) {
super(
// Reviewers aren't generally associated with a particular patch set
// (although as an implementation detail they were in ReviewDb). Just
// use the latest patch set at the time of the event.
null,
reviewer.getColumnKey(),
// TODO(dborowitz): Real account ID shouldn't really matter for
// reviewers, but we might have to deal with this to avoid ChangeBundle
// diffs when run against real data.
reviewer.getColumnKey(),
reviewer.getValue(),
changeCreatedOn,
null);
this.reviewer = reviewer;
}
@Override
boolean uniquePerUpdate() {
return false;
}
@Override
void apply(ChangeUpdate update) throws IOException, OrmException {
checkUpdate(update);
update.putReviewer(reviewer.getColumnKey(), reviewer.getRowKey());
}
@Override
protected void addToString(ToStringHelper helper) {
helper.add("account", reviewer.getColumnKey()).add("state", reviewer.getRowKey());
}
}

3
java/com/google/gerrit/server/schema/DatabaseModule.java
View File

@@ -16,8 +16,6 @@ package com.google.gerrit.server.schema;
import com.google.gerrit.extensions.config.FactoryModule;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.notedb.ChangeBundleReader;
import com.google.gerrit.server.notedb.GwtormChangeBundleReader;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Key;
@@ -35,6 +33,5 @@ public class DatabaseModule extends FactoryModule {
() -> {
throw new OrmException("ReviewDb no longer exists");
});
bind(ChangeBundleReader.class).to(GwtormChangeBundleReader.class);
}
}

3
java/com/google/gerrit/testing/InMemoryModule.java
View File

@@ -73,8 +73,6 @@ import com.google.gerrit.server.index.group.AllGroupsIndexer;
import com.google.gerrit.server.index.group.GroupIndexCollection;
import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier;
import com.google.gerrit.server.notedb.ChangeBundleReader;
import com.google.gerrit.server.notedb.GwtormChangeBundleReader;
import com.google.gerrit.server.notedb.MutableNotesMigration;
import com.google.gerrit.server.notedb.NotesMigration;
import com.google.gerrit.server.patch.DiffExecutor;
@@ -198,7 +196,6 @@ public class InMemoryModule extends FactoryModule {
bind(ListeningExecutorService.class)
.annotatedWith(ChangeUpdateExecutor.class)
.toInstance(MoreExecutors.newDirectExecutorService());
bind(ChangeBundleReader.class).to(GwtormChangeBundleReader.class);
bind(SecureStore.class).to(DefaultSecureStore.class);
TypeLiteral<SchemaFactory<ReviewDb>> schemaFactory =

225
java/com/google/gerrit/testing/NoteDbChecker.java
View File

@@ -1,225 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.testing;
import static com.google.common.truth.Truth.assertThat;
import static java.util.Comparator.comparing;
import static java.util.stream.Collectors.toList;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.reviewdb.server.ReviewDbUtil;
import com.google.gerrit.server.CommentsUtil;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.notedb.ChangeBundle;
import com.google.gerrit.server.notedb.ChangeBundleReader;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.notedb.MutableNotesMigration;
import com.google.gerrit.server.notedb.rebuild.ChangeRebuilder;
import com.google.gwtorm.client.IntKey;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.OrmRuntimeException;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Stream;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Repository;
import org.junit.runner.Description;
@Singleton
public class NoteDbChecker {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final Provider<ReviewDb> dbProvider;
private final GitRepositoryManager repoManager;
private final MutableNotesMigration notesMigration;
private final ChangeBundleReader bundleReader;
private final ChangeNotes.Factory notesFactory;
private final ChangeRebuilder changeRebuilder;
private final CommentsUtil commentsUtil;
@Inject
NoteDbChecker(
Provider<ReviewDb> dbProvider,
GitRepositoryManager repoManager,
MutableNotesMigration notesMigration,
ChangeBundleReader bundleReader,
ChangeNotes.Factory notesFactory,
ChangeRebuilder changeRebuilder,
CommentsUtil commentsUtil) {
this.dbProvider = dbProvider;
this.repoManager = repoManager;
this.bundleReader = bundleReader;
this.notesMigration = notesMigration;
this.notesFactory = notesFactory;
this.changeRebuilder = changeRebuilder;
this.commentsUtil = commentsUtil;
}
public void rebuildAndCheckAllChanges() throws Exception {
rebuildAndCheckChanges(
getUnwrappedDb().changes().all().toList().stream().map(Change::getId),
ImmutableListMultimap.of());
}
public void rebuildAndCheckChanges(Change.Id... changeIds) throws Exception {
rebuildAndCheckChanges(Arrays.stream(changeIds), ImmutableListMultimap.of());
}
private void rebuildAndCheckChanges(
Stream<Change.Id> changeIds, ListMultimap<Change.Id, String> expectedDiffs) throws Exception {
ReviewDb db = getUnwrappedDb();
List<ChangeBundle> allExpected = readExpected(changeIds);
boolean oldWrite = notesMigration.rawWriteChangesSetting();
boolean oldRead = notesMigration.readChanges();
try {
notesMigration.setWriteChanges(true);
notesMigration.setReadChanges(true);
List<String> msgs = new ArrayList<>();
for (ChangeBundle expected : allExpected) {
Change c = expected.getChange();
try {
changeRebuilder.rebuild(db, c.getId());
} catch (RepositoryNotFoundException e) {
msgs.add("Repository not found for change, cannot convert: " + c);
}
}
checkActual(allExpected, expectedDiffs, msgs);
} finally {
notesMigration.setReadChanges(oldRead);
notesMigration.setWriteChanges(oldWrite);
}
}
public void checkChanges(Change.Id... changeIds) throws Exception {
checkActual(
readExpected(Arrays.stream(changeIds)), ImmutableListMultimap.of(), new ArrayList<>());
}
public void rebuildAndCheckChange(Change.Id changeId, String... expectedDiff) throws Exception {
ImmutableListMultimap.Builder<Change.Id, String> b = ImmutableListMultimap.builder();
b.putAll(changeId, Arrays.asList(expectedDiff));
rebuildAndCheckChanges(Stream.of(changeId), b.build());
}
public void assertNoChangeRef(Project.NameKey project, Change.Id changeId) throws Exception {
try (Repository repo = repoManager.openRepository(project)) {
assertThat(repo.exactRef(RefNames.changeMetaRef(changeId))).isNull();
}
}
public void assertNoReviewDbChanges(Description desc) throws Exception {
ReviewDb db = getUnwrappedDb();
assertThat(db.changes().all().toList()).named("Changes in " + desc.getTestClass()).isEmpty();
assertThat(db.changeMessages().all().toList())
.named("ChangeMessages in " + desc.getTestClass())
.isEmpty();
assertThat(db.patchSets().all().toList())
.named("PatchSets in " + desc.getTestClass())
.isEmpty();
assertThat(db.patchSetApprovals().all().toList())
.named("PatchSetApprovals in " + desc.getTestClass())
.isEmpty();
assertThat(db.patchComments().all().toList())
.named("PatchLineComments in " + desc.getTestClass())
.isEmpty();
}
private List<ChangeBundle> readExpected(Stream<Change.Id> changeIds) throws Exception {
boolean old = notesMigration.readChanges();
try {
notesMigration.setReadChanges(false);
return changeIds
.sorted(comparing(IntKey::get))
.map(this::readBundleUnchecked)
.collect(toList());
} finally {
notesMigration.setReadChanges(old);
}
}
private ChangeBundle readBundleUnchecked(Change.Id id) {
try {
return bundleReader.fromReviewDb(getUnwrappedDb(), id);
} catch (OrmException e) {
throw new OrmRuntimeException(e);
}
}
private void checkActual(
List<ChangeBundle> allExpected,
ListMultimap<Change.Id, String> expectedDiffs,
List<String> msgs)
throws Exception {
ReviewDb db = getUnwrappedDb();
boolean oldRead = notesMigration.readChanges();
boolean oldWrite = notesMigration.rawWriteChangesSetting();
try {
notesMigration.setWriteChanges(true);
notesMigration.setReadChanges(true);
for (ChangeBundle expected : allExpected) {
Change c = expected.getChange();
ChangeBundle actual;
try {
actual =
ChangeBundle.fromNotes(
commentsUtil, notesFactory.create(db, c.getProject(), c.getId()));
} catch (Throwable t) {
String msg = "Error converting change: " + c;
msgs.add(msg);
logger.atSevere().withCause(t).log(msg);
continue;
}
List<String> diff = expected.differencesFrom(actual);
List<String> expectedDiff = expectedDiffs.get(c.getId());
if (!diff.equals(expectedDiff)) {
msgs.add("Differences between ReviewDb and NoteDb for " + c + ":");
msgs.addAll(diff);
if (!expectedDiff.isEmpty()) {
msgs.add("Expected differences:");
msgs.addAll(expectedDiff);
}
msgs.add("");
} else {
System.err.println("NoteDb conversion of change " + c.getId() + " successful");
}
}
} finally {
notesMigration.setReadChanges(oldRead);
notesMigration.setWriteChanges(oldWrite);
}
if (!msgs.isEmpty()) {
throw new AssertionError(Joiner.on('\n').join(msgs));
}
}
private ReviewDb getUnwrappedDb() {
ReviewDb db = dbProvider.get();
return ReviewDbUtil.unwrapDb(db);
}
}

3
javatests/com/google/gerrit/acceptance/api/accounts/AccountIT.java
View File

@@ -123,7 +123,6 @@ import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
import com.google.gerrit.server.git.meta.MetaDataUpdate;
import com.google.gerrit.server.index.account.AccountIndexer;
import com.google.gerrit.server.index.account.StalenessChecker;
import com.google.gerrit.server.notedb.rebuild.ChangeRebuilderImpl;
import com.google.gerrit.server.project.ProjectConfig;
import com.google.gerrit.server.project.RefPattern;
import com.google.gerrit.server.query.account.InternalAccountQuery;
@@ -468,7 +467,7 @@ public class AccountIT extends AbstractDaemonTest {
RevCommit c = rw.parseCommit(ref.getObjectId());
long timestampDiffMs =
Math.abs(c.getCommitTime() * 1000L - getAccount(accountId).getRegisteredOn().getTime());
assertThat(timestampDiffMs).isAtMost(ChangeRebuilderImpl.MAX_WINDOW_MS);
assertThat(timestampDiffMs).isAtMost(SECONDS.toMillis(1));
// Check the 'account.config' file.
try (TreeWalk tw = TreeWalk.forPath(or, AccountProperties.ACCOUNT_CONFIG, c.getTree())) {

7
javatests/com/google/gerrit/server/notedb/AbstractChangeNotesTest.java
View File

@@ -137,7 +137,7 @@ public abstract class AbstractChangeNotesTest extends GerritBaseTests {
install(new GitModule());
install(new DefaultUrlFormatter.Module());
install(NoteDbModule.forTest(testConfig));
install(NoteDbModule.forTest());
bind(AllUsersName.class).toProvider(AllUsersNameProvider.class);
bind(String.class).annotatedWith(GerritServerId.class).toInstance("gerrit");
bind(GitRepositoryManager.class).toInstance(repoManager);
@@ -172,11 +172,6 @@ public abstract class AbstractChangeNotesTest extends GerritBaseTests {
() -> {
throw new UnsupportedOperationException();
});
bind(ChangeBundleReader.class)
.toInstance(
(db, id) -> {
throw new UnsupportedOperationException();
});
}
});

1976
javatests/com/google/gerrit/server/notedb/ChangeBundleTest.java
View File

File diff suppressed because it is too large Load Diff

232
javatests/com/google/gerrit/server/notedb/rebuild/EventSorterTest.java
View File

@@ -1,232 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb.rebuild;
import static com.google.common.truth.Truth.assertThat;
import static java.util.stream.Collectors.toList;
import static org.junit.Assert.fail;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gerrit.server.util.time.TimeUtil;
import com.google.gerrit.testing.GerritBaseTests;
import com.google.gerrit.testing.TestTimeUtil;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import org.junit.Before;
import org.junit.Test;
public class EventSorterTest extends GerritBaseTests {
private class TestEvent extends Event {
protected TestEvent(Timestamp when) {
super(
new PatchSet.Id(new Change.Id(1), 1),
new Account.Id(1000),
new Account.Id(1000),
when,
changeCreatedOn,
null);
}
@Override
boolean uniquePerUpdate() {
return false;
}
@Override
void apply(ChangeUpdate update) {
throw new UnsupportedOperationException();
}
@SuppressWarnings("deprecation")
@Override
public String toString() {
return "E{" + when.getSeconds() + '}';
}
}
private Timestamp changeCreatedOn;
@Before
public void setUp() {
TestTimeUtil.resetWithClockStep(10, TimeUnit.SECONDS);
changeCreatedOn = TimeUtil.nowTs();
}
@Test
public void naturalSort() {
Event e1 = new TestEvent(TimeUtil.nowTs());
Event e2 = new TestEvent(TimeUtil.nowTs());
Event e3 = new TestEvent(TimeUtil.nowTs());
for (List<Event> events : Collections2.permutations(events(e1, e2, e3))) {
assertSorted(events, events(e1, e2, e3));
}
}
@Test
public void topoSortOneDep() {
List<Event> es;
// Input list is 0,1,2
// 0 depends on 1 => 1,0,2
es = threeEventsOneDep(0, 1);
assertSorted(es, events(es, 1, 0, 2));
// 1 depends on 0 => 0,1,2
es = threeEventsOneDep(1, 0);
assertSorted(es, events(es, 0, 1, 2));
// 0 depends on 2 => 1,2,0
es = threeEventsOneDep(0, 2);
assertSorted(es, events(es, 1, 2, 0));
// 2 depends on 0 => 0,1,2
es = threeEventsOneDep(2, 0);
assertSorted(es, events(es, 0, 1, 2));
// 1 depends on 2 => 0,2,1
es = threeEventsOneDep(1, 2);
assertSorted(es, events(es, 0, 2, 1));
// 2 depends on 1 => 0,1,2
es = threeEventsOneDep(2, 1);
assertSorted(es, events(es, 0, 1, 2));
}
private List<Event> threeEventsOneDep(int depFromIdx, int depOnIdx) {
List<Event> events =
Lists.newArrayList(
new TestEvent(TimeUtil.nowTs()),
new TestEvent(TimeUtil.nowTs()),
new TestEvent(TimeUtil.nowTs()));
events.get(depFromIdx).addDep(events.get(depOnIdx));
return events;
}
@Test
public void lastEventDependsOnFirstEvent() {
List<Event> events = new ArrayList<>();
for (int i = 0; i < 20; i++) {
events.add(new TestEvent(TimeUtil.nowTs()));
}
events.get(events.size() - 1).addDep(events.get(0));
assertSorted(events, events);
}
@Test
public void firstEventDependsOnLastEvent() {
List<Event> events = new ArrayList<>();
for (int i = 0; i < 20; i++) {
events.add(new TestEvent(TimeUtil.nowTs()));
}
events.get(0).addDep(events.get(events.size() - 1));
List<Event> expected = new ArrayList<>();
expected.addAll(events.subList(1, events.size()));
expected.add(events.get(0));
assertSorted(events, expected);
}
@Test
public void topoSortChainOfDeps() {
Event e1 = new TestEvent(TimeUtil.nowTs());
Event e2 = new TestEvent(TimeUtil.nowTs());
Event e3 = new TestEvent(TimeUtil.nowTs());
Event e4 = new TestEvent(TimeUtil.nowTs());
e1.addDep(e2);
e2.addDep(e3);
e3.addDep(e4);
assertSorted(events(e1, e2, e3, e4), events(e4, e3, e2, e1));
}
@Test
public void topoSortMultipleDeps() {
Event e1 = new TestEvent(TimeUtil.nowTs());
Event e2 = new TestEvent(TimeUtil.nowTs());
Event e3 = new TestEvent(TimeUtil.nowTs());
Event e4 = new TestEvent(TimeUtil.nowTs());
e1.addDep(e2);
e1.addDep(e4);
e2.addDep(e3);
// Processing 3 pops 2, processing 4 pops 1.
assertSorted(events(e2, e3, e1, e4), events(e3, e2, e4, e1));
}
@Test
public void topoSortMultipleDepsPreservesNaturalOrder() {
Event e1 = new TestEvent(TimeUtil.nowTs());
Event e2 = new TestEvent(TimeUtil.nowTs());
Event e3 = new TestEvent(TimeUtil.nowTs());
Event e4 = new TestEvent(TimeUtil.nowTs());
e1.addDep(e4);
e2.addDep(e4);
e3.addDep(e4);
// Processing 4 pops 1, 2, 3 in natural order.
assertSorted(events(e4, e3, e2, e1), events(e4, e1, e2, e3));
}
@Test
public void topoSortCycle() {
Event e1 = new TestEvent(TimeUtil.nowTs());
Event e2 = new TestEvent(TimeUtil.nowTs());
// Implementation is not really defined, but infinite looping would be bad.
// According to current implementation details, 2 pops 1, 1 pops 2 which was
// already seen.
assertSorted(events(e2, e1), events(e1, e2));
}
@Test
public void topoSortDepNotInInputList() {
Event e1 = new TestEvent(TimeUtil.nowTs());
Event e2 = new TestEvent(TimeUtil.nowTs());
Event e3 = new TestEvent(TimeUtil.nowTs());
e1.addDep(e3);
List<Event> events = events(e2, e1);
try {
new EventSorter(events).sort();
fail("expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
// Expected.
}
}
private static List<Event> events(Event... es) {
return Lists.newArrayList(es);
}
private static List<Event> events(List<Event> in, Integer... indexes) {
return Stream.of(indexes).map(in::get).collect(toList());
}
private static void assertSorted(List<Event> unsorted, List<Event> expected) {
List<Event> actual = new ArrayList<>(unsorted);
new EventSorter(actual).sort();
assertThat(actual).named("sorted" + unsorted).isEqualTo(expected);
}
}