Implement staleness checker for index changes

As described in I622dbbb3, we use a combination of the change row
version, the ref states, and ref patterns stored in the index to check
whether the change document stored in the secondary index is up to
date with the primary storage.

Change-Id: I8d4a63b705527c5b774373f75dbc28ca9fd10158

gerrit-server/src/main/java/com/google/gerrit/server/index/change/StalenessChecker.java

@@ -20,25 +20,133 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.stream.Collectors.joining;
 
 import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Multimap;
 import com.google.common.collect.SetMultimap;
+import com.google.common.collect.Sets;
 import com.google.gerrit.common.Nullable;
+import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.reviewdb.server.ReviewDb;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.gerrit.server.index.IndexConfig;
+import com.google.gerrit.server.notedb.ChangeNotes;
+import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
+import com.google.gerrit.server.query.change.ChangeData;
+import com.google.gwtorm.server.OrmException;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
 
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.Repository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.Optional;
+import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.StreamSupport;
 
+@Singleton
 public class StalenessChecker {
-  public static SetMultimap<Project.NameKey, RefState> parseStates(
+  private static final Logger log =
+      LoggerFactory.getLogger(StalenessChecker.class);
+
+  private final ImmutableSet<String> FIELDS = ImmutableSet.of(
+      ChangeField.CHANGE.getName(),
+      ChangeField.REF_STATE.getName(),
+      ChangeField.REF_STATE_PATTERN.getName());
+
+  private final ChangeIndexCollection indexes;
+  private final GitRepositoryManager repoManager;
+  private final IndexConfig indexConfig;
+  private final Provider<ReviewDb> db;
+
+  @Inject
+  StalenessChecker(
+      ChangeIndexCollection indexes,
+      GitRepositoryManager repoManager,
+      IndexConfig indexConfig,
+      Provider<ReviewDb> db) {
+    this.indexes = indexes;
+    this.repoManager = repoManager;
+    this.indexConfig = indexConfig;
+    this.db = db;
+  }
+
+  boolean isStale(Change.Id id) throws IOException, OrmException {
+    ChangeIndex i = indexes.getSearchIndex();
+    if (i == null) {
+      return false; // No index; caller couldn't do anything if it is stale.
+    }
+    if (!i.getSchema().hasField(ChangeField.REF_STATE)
+        || !i.getSchema().hasField(ChangeField.REF_STATE_PATTERN)) {
+      return false; // Index version not new enough for this check.
+    }
+
+    Optional<ChangeData> result = i.get(
+        id, IndexedChangeQuery.createOptions(indexConfig, 0, 1, FIELDS));
+    if (!result.isPresent()) {
+      return true; // Not in index, but caller wants it to be.
+    }
+    ChangeData cd = result.get();
+    if (reviewDbChangeIsStale(
+        cd.change(),
+        ChangeNotes.readOneReviewDbChange(db.get(), cd.getId()))) {
+      return true;
+    }
+
+    return isStale(repoManager, id, parseStates(cd), parsePatterns(cd));
+  }
+
+  @VisibleForTesting
+  static boolean isStale(GitRepositoryManager repoManager,
+      Change.Id id,
+      SetMultimap<Project.NameKey, RefState> states,
+      Multimap<Project.NameKey, RefStatePattern> patterns) {
+    Set<Project.NameKey> projects =
+        Sets.union(states.keySet(), patterns.keySet());
+
+    for (Project.NameKey p : projects) {
+      if (isStale(repoManager, id, p, states, patterns)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  @VisibleForTesting
+  static boolean reviewDbChangeIsStale(
+      Change indexChange, @Nullable Change reviewDbChange) {
+    if (reviewDbChange == null) {
+      return false; // Nothing the caller can do.
+    }
+    checkArgument(indexChange.getId().equals(reviewDbChange.getId()),
+        "mismatched change ID: %s != %s",
+        indexChange.getId(), reviewDbChange.getId());
+    if (PrimaryStorage.of(reviewDbChange) != PrimaryStorage.REVIEW_DB) {
+      return false; // Not a ReviewDb change, don't check rowVersion.
+    }
+    return reviewDbChange.getRowVersion() != indexChange.getRowVersion();
+  }
+
+  private SetMultimap<Project.NameKey, RefState> parseStates(ChangeData cd) {
+    return parseStates(cd.getRefStates());
+  }
+
+  @VisibleForTesting
+  static SetMultimap<Project.NameKey, RefState> parseStates(
       Iterable<byte[]> states) {
     RefState.check(states != null, null);
     SetMultimap<Project.NameKey, RefState> result = HashMultimap.create();
@@ -58,6 +166,11 @@ public class StalenessChecker {
     return result;
   }
 
+  private Multimap<Project.NameKey, RefStatePattern> parsePatterns(
+      ChangeData cd) {
+    return parsePatterns(cd.getRefStatePatterns());
+  }
+
   public static ListMultimap<Project.NameKey, RefStatePattern> parsePatterns(
       Iterable<byte[]> patterns) {
     RefStatePattern.check(patterns != null, null);
@@ -75,6 +188,31 @@ public class StalenessChecker {
     return result;
   }
 
+  private static boolean isStale(GitRepositoryManager repoManager,
+      Change.Id id, Project.NameKey project,
+      SetMultimap<Project.NameKey, RefState> allStates,
+      Multimap<Project.NameKey, RefStatePattern> allPatterns) {
+    try (Repository repo = repoManager.openRepository(project)) {
+      Set<RefState> states = allStates.get(project);
+      for (RefState state : states) {
+        if (!state.match(repo)) {
+          return true;
+        }
+      }
+      for (RefStatePattern pattern : allPatterns.get(project)) {
+        if (!pattern.match(repo, states)) {
+          return true;
+        }
+      }
+      return false;
+    } catch (IOException e) {
+      log.warn(
+          String.format("error checking staleness of %s in %s", id, project),
+          e);
+      return true;
+    }
+  }
+
   @AutoValue
   public abstract static class RefState {
     static RefState create(String ref, String sha) {
@@ -106,6 +244,12 @@ public class StalenessChecker {
 
     abstract String ref();
     abstract ObjectId id();
+
+    private boolean match(Repository repo) throws IOException {
+      Ref ref = repo.exactRef(ref());
+      ObjectId expected = ref != null ? ref.getObjectId() : ObjectId.zeroId();
+      return id().equals(expected);
+    }
   }
 
   /**
@@ -147,5 +291,18 @@ public class StalenessChecker {
     boolean match(String refName) {
       return regex().matcher(refName).find();
     }
+
+    private boolean match(Repository repo, Set<RefState> expected)
+        throws IOException {
+      for (Ref r : repo.getRefDatabase().getRefs(prefix()).values()) {
+        if (!match(r.getName())) {
+          continue;
+        }
+        if (!expected.contains(RefState.of(r))) {
+          return false;
+        }
+      }
+      return true;
+    }
   }
 }