Store in index the state of all refs that contribute to a change

The primary storage (ReviewDb or NoteDb) and the secondary index
(Lucene, etc.) are not updated atomically, so they can get out of sync
in various ways. For example, the server may crash before writing to
the index, or index updates for concurrent primary storage writes may
happen out of order. We want to be able to detect when this has
happened.

One component of the state of a change is the row version field in the
ReviewDb entity; this is already stored in the index in the serialized
Change entity. The remainder of a change's state comprises various
refs:

 * NoteDb refs, if NoteDb is the primary storage:
    * Change meta ref.
    * Draft comment refs.
    * Robot comment ref.
 * Edit refs in the project repo.
 * Star refs in All-Users.

Store each of these in the index in a field REF_STATE, with the simple
format "<project>:<ref>:<sha>".

In addition to storing the refs that *are* present, we also need to
keep track of which refs are *not* present. For example, if only one
user is listed in the EDITBY field, then there must exist an edit ref
for that user and only that user, and no other users. This is
represented by storing a list of "<project>:<pattern>" values in the
REF_STATE_PATTERN field. A change is up to date only if all refs in
each <project> matching each <pattern> are also listed explicitly in
REF_STATE. The pattern format is a slightly extended git refspec
pattern which allows arbitrarily many '*'s, to deal with the tricky
case of "refs/users/UU/UUUU/edit-C/P" => "refs/users/*/edit-C/*".

This change implements the REF_STATE and REF_STATE_PATTERN change
fields, which require some additional ChangeData methods, as well as
structured data types for these fields. These data types live in a new
class StalenessChecker, which will be later extended to use them to
check for staleness.

Change-Id: I622dbbb334b5aca4d58f770ad453e034a5bc9529

gerrit-server/src/main/java/com/google/gerrit/server/index/FieldDef.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.base.CharMatcher;
 import com.google.common.base.Preconditions;
+import com.google.gerrit.server.config.AllUsersName;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.TrackingFooters;
 import com.google.gwtorm.server.OrmException;
@@ -65,14 +66,17 @@ public abstract class FieldDef<I, T> {
   public static class FillArgs {
     public final TrackingFooters trackingFooters;
     public final boolean allowsDrafts;
+    public final AllUsersName allUsers;
 
     @Inject
     FillArgs(TrackingFooters trackingFooters,
-        @GerritServerConfig Config cfg) {
+        @GerritServerConfig Config cfg,
+        AllUsersName allUsers) {
       this.trackingFooters = trackingFooters;
       this.allowsDrafts = cfg == null
           ? true
           : cfg.getBoolean("change", "allowDrafts", true);
+      this.allUsers = allUsers;
     }
   }
 

gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeField.java

@@ -36,13 +36,20 @@ import com.google.gerrit.reviewdb.client.ChangeMessage;
 import com.google.gerrit.reviewdb.client.Comment;
 import com.google.gerrit.reviewdb.client.PatchSet;
 import com.google.gerrit.reviewdb.client.PatchSetApproval;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.reviewdb.client.RefNames;
 import com.google.gerrit.server.OutputFormat;
 import com.google.gerrit.server.ReviewerSet;
 import com.google.gerrit.server.StarredChangesUtil;
 import com.google.gerrit.server.index.FieldDef;
 import com.google.gerrit.server.index.FieldType;
 import com.google.gerrit.server.index.SchemaUtil;
+import com.google.gerrit.server.index.change.StalenessChecker.RefState;
+import com.google.gerrit.server.index.change.StalenessChecker.RefStatePattern;
+import com.google.gerrit.server.notedb.ChangeNotes;
+import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
 import com.google.gerrit.server.notedb.ReviewerStateInternal;
+import com.google.gerrit.server.notedb.RobotCommentNotes;
 import com.google.gerrit.server.project.SubmitRuleOptions;
 import com.google.gerrit.server.query.change.ChangeData;
 import com.google.gerrit.server.query.change.ChangeQueryBuilder;
@@ -945,6 +952,74 @@ public class ChangeField {
     return result;
   }
 
+  /**
+   * All values of all refs that were used in the course of indexing this
+   * document.
+   * <p>
+   * Emitted as UTF-8 encoded strings of the form
+   * {@code project:ref/name:[hex sha]}.
+   */
+  public static final FieldDef<ChangeData, Iterable<byte[]>> REF_STATE =
+      new FieldDef.Repeatable<ChangeData, byte[]>(
+          "ref_state", FieldType.STORED_ONLY, true) {
+        @Override
+        public Iterable<byte[]> get(ChangeData input, FillArgs args)
+            throws OrmException {
+          List<byte[]> result = new ArrayList<>();
+          Project.NameKey project = input.change().getProject();
+
+          input.editRefs().values().forEach(
+              r -> result.add(RefState.of(r).toByteArray(project)));
+          input.starRefs().values().forEach(
+              r -> result.add(RefState.of(r.ref()).toByteArray(project)));
+
+          if (PrimaryStorage.of(input.change()) == PrimaryStorage.NOTE_DB) {
+            ChangeNotes notes = input.notes();
+            result.add(RefState.create(notes.getRefName(), notes.getMetaId())
+                .toByteArray(project));
+            notes.getRobotComments(); // Force loading robot comments.
+            RobotCommentNotes robotNotes = notes.getRobotCommentNotes();
+            result.add(
+                RefState.create(robotNotes.getRefName(), robotNotes.getMetaId())
+                    .toByteArray(project));
+            input.draftRefs().values().forEach(
+                r -> result.add(RefState.of(r).toByteArray(args.allUsers)));
+          }
+
+          return result;
+        } };
+
+  /**
+   * All ref wildcard patterns that were used in the course of indexing this
+   * document.
+   * <p>
+   * Emitted as UTF-8 encoded strings of the form {@code project:ref/name/*}.
+   * See {@link RefStatePattern} for the pattern format.
+   */
+  public static final FieldDef<ChangeData, Iterable<byte[]>>
+      REF_STATE_PATTERN = new FieldDef.Repeatable<ChangeData, byte[]>(
+          "ref_state_pattern", FieldType.STORED_ONLY, true) {
+    @Override
+    public Iterable<byte[]> get(ChangeData input, FillArgs args)
+        throws OrmException {
+      Change.Id id = input.getId();
+      Project.NameKey project = input.change().getProject();
+      List<byte[]> result = new ArrayList<>(3);
+      result.add(RefStatePattern.create(
+              RefNames.REFS_USERS + "*/" + RefNames.EDIT_PREFIX + id + "/*")
+          .toByteArray(project));
+      if (PrimaryStorage.of(input.change()) == PrimaryStorage.NOTE_DB) {
+        result.add(
+            RefStatePattern.create(RefNames.refsStarredChangesPrefix(id) + "*")
+                .toByteArray(args.allUsers));
+        result.add(
+            RefStatePattern.create(RefNames.refsDraftCommentsPrefix(id) + "*")
+                .toByteArray(args.allUsers));
+      }
+      return result;
+    }
+  };
+
   public static final Integer NOT_REVIEWED = -1;
 
   private static String getTopic(ChangeData input) throws OrmException {

gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemaDefinitions.java

@@ -73,12 +73,18 @@ public class ChangeSchemaDefinitions extends SchemaDefinitions<ChangeData> {
       .add(ChangeField.LABEL2)
       .build();
 
+  @Deprecated
   static final Schema<ChangeData> V35 =
       schema(V34,
           ChangeField.SUBMIT_RECORD,
           ChangeField.STORED_SUBMIT_RECORD_LENIENT,
           ChangeField.STORED_SUBMIT_RECORD_STRICT);
 
+  static final Schema<ChangeData> V36 =
+      schema(V35,
+          ChangeField.REF_STATE,
+          ChangeField.REF_STATE_PATTERN);
+
   public static final String NAME = "changes";
   public static final ChangeSchemaDefinitions INSTANCE =
       new ChangeSchemaDefinitions();

gerrit-server/src/main/java/com/google/gerrit/server/index/change/StalenessChecker.java

@@ -0,0 +1,151 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index.change;
+
+import static com.google.common.base.MoreObjects.firstNonNull;
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.stream.Collectors.joining;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.SetMultimap;
+import com.google.gerrit.common.Nullable;
+import com.google.gerrit.reviewdb.client.Project;
+
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.Repository;
+
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.stream.StreamSupport;
+
+public class StalenessChecker {
+  public static SetMultimap<Project.NameKey, RefState> parseStates(
+      Iterable<byte[]> states) {
+    RefState.check(states != null, null);
+    SetMultimap<Project.NameKey, RefState> result = HashMultimap.create();
+    for (byte[] b : states) {
+      RefState.check(b != null, null);
+      String s = new String(b, UTF_8);
+      List<String> parts = Splitter.on(':').splitToList(s);
+      RefState.check(
+          parts.size() == 3
+              && !parts.get(0).isEmpty()
+              && !parts.get(1).isEmpty(),
+          s);
+      result.put(
+          new Project.NameKey(parts.get(0)),
+          RefState.create(parts.get(1), parts.get(2)));
+    }
+    return result;
+  }
+
+  public static ListMultimap<Project.NameKey, RefStatePattern> parsePatterns(
+      Iterable<byte[]> patterns) {
+    RefStatePattern.check(patterns != null, null);
+    ListMultimap<Project.NameKey, RefStatePattern> result =
+        ArrayListMultimap.create();
+    for (byte[] b : patterns) {
+      RefStatePattern.check(b != null, null);
+      String s = new String(b, UTF_8);
+      List<String> parts = Splitter.on(':').splitToList(s);
+      RefStatePattern.check(parts.size() == 2, s);
+      result.put(
+          new Project.NameKey(parts.get(0)),
+          RefStatePattern.create(parts.get(1)));
+    }
+    return result;
+  }
+
+  @AutoValue
+  public abstract static class RefState {
+    static RefState create(String ref, String sha) {
+      return new AutoValue_StalenessChecker_RefState(
+          ref, ObjectId.fromString(sha));
+    }
+
+    static RefState create(String ref, @Nullable ObjectId id) {
+      return new AutoValue_StalenessChecker_RefState(
+          ref, firstNonNull(id, ObjectId.zeroId()));
+    }
+
+    static RefState of(Ref ref) {
+      return new AutoValue_StalenessChecker_RefState(
+          ref.getName(), ref.getObjectId());
+    }
+
+    byte[] toByteArray(Project.NameKey project) {
+      byte[] a = (project.toString() + ':' + ref() + ':').getBytes(UTF_8);
+      byte[] b = new byte[a.length + Constants.OBJECT_ID_STRING_LENGTH];
+      System.arraycopy(a, 0, b, 0, a.length);
+      id().copyTo(b, a.length);
+      return b;
+    }
+
+    private static void check(boolean condition, String str) {
+      checkArgument(condition, "invalid RefState: %s", str);
+    }
+
+    abstract String ref();
+    abstract ObjectId id();
+  }
+
+  /**
+   * Pattern for matching refs.
+   * <p>
+   * Similar to '*' syntax for native Git refspecs, but slightly more powerful:
+   * the pattern may contain arbitrarily many asterisks. There must be at least
+   * one '*' and the first one must immediately follow a '/'.
+   */
+  @AutoValue
+  public abstract static class RefStatePattern {
+    static RefStatePattern create(String pattern) {
+      int star = pattern.indexOf('*');
+      check(star > 0 && pattern.charAt(star - 1) == '/', pattern);
+      String prefix = pattern.substring(0, star);
+      check(Repository.isValidRefName(pattern.replace('*', 'x')), pattern);
+
+      // Quote everything except the '*'s, which become ".*".
+      String regex =
+          StreamSupport.stream(Splitter.on('*').split(pattern).spliterator(), false)
+              .map(Pattern::quote)
+              .collect(joining(".*", "^", "$"));
+      return new AutoValue_StalenessChecker_RefStatePattern(
+          pattern, prefix, Pattern.compile(regex));
+    }
+
+    byte[] toByteArray(Project.NameKey project) {
+      return (project.toString() + ':' + pattern()).getBytes(UTF_8);
+    }
+
+    private static void check(boolean condition, String str) {
+      checkArgument(condition, "invalid RefStatePattern: %s", str);
+    }
+
+    abstract String pattern();
+    abstract String prefix();
+    abstract Pattern regex();
+
+    boolean match(String refName) {
+      return regex().matcher(refName).find();
+    }
+  }
+}