Add ref_states to ProjectFields

This commit adds a new field called ref_states to the ProjectIndex. This
field can be used to perform staleness detection based on NoteDb ref
states. We store the HEAD SHA1s in 40 digit hex notation for the project
in question as well as all of it's ancestors.

Storing the ancestors ref states as well is necessary because the index
stores the names of all ancestors, so we need to detect when the
project's hirarchy changes and we get out of sync.

This approach follows what was done for changes, accounts and groups.

A subsequent commit will build a staleness checker and wire it up.

Change-Id: I08efd4286b4fd5c577c0863802bcdf051972f5cf

java/com/google/gerrit/acceptance/ProjectResetter.java

@@ -23,6 +23,7 @@ import com.google.common.collect.Multimap;
 import com.google.common.collect.MultimapBuilder;
 import com.google.common.collect.Sets;
 import com.google.gerrit.common.Nullable;
+import com.google.gerrit.index.RefState;
 import com.google.gerrit.reviewdb.client.Account;
 import com.google.gerrit.reviewdb.client.AccountGroup;
 import com.google.gerrit.reviewdb.client.Project;
@@ -32,7 +33,6 @@ import com.google.gerrit.server.account.GroupCache;
 import com.google.gerrit.server.account.GroupIncludeCache;
 import com.google.gerrit.server.config.AllUsersName;
 import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.index.RefState;
 import com.google.gerrit.server.index.account.AccountIndexer;
 import com.google.gerrit.server.index.group.GroupIndexer;
 import com.google.gerrit.server.project.ProjectCache;

java/com/google/gerrit/index/BUILD

@@ -37,6 +37,7 @@ java_library(
         "//java/com/google/gerrit/common:annotations",
         "//java/com/google/gerrit/extensions:api",
         "//java/com/google/gerrit/metrics",
+        "//java/com/google/gerrit/reviewdb:server",
         "//lib:guava",
         "//lib:gwtjsonrpc",
         "//lib:gwtorm",

java/com/google/gerrit/index/RefState.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.gerrit.server.index;
+package com.google.gerrit.index;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkArgument;

java/com/google/gerrit/index/project/ProjectData.java

@@ -14,23 +14,43 @@
 
 package com.google.gerrit.index.project;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
+
 import com.google.common.collect.ImmutableList;
 import com.google.gerrit.reviewdb.client.Project;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
 
 public class ProjectData {
   private final Project project;
-  private final ImmutableList<Project.NameKey> ancestors;
+  private final Optional<ProjectData> parent;
 
-  public ProjectData(Project project, Iterable<Project.NameKey> ancestors) {
+  public ProjectData(Project project, Optional<ProjectData> parent) {
     this.project = project;
-    this.ancestors = ImmutableList.copyOf(ancestors);
+    this.parent = parent;
   }
 
   public Project getProject() {
     return project;
   }
 
-  public ImmutableList<Project.NameKey> getAncestors() {
-    return ancestors;
+  public Optional<ProjectData> getParent() {
+    return parent;
+  }
+
+  /** Returns all {@link ProjectData} in the hierarchy starting with the current one. */
+  public ImmutableList<ProjectData> tree() {
+    List<ProjectData> parents = new ArrayList<>();
+    Optional<ProjectData> curr = Optional.of(this);
+    while (curr.isPresent()) {
+      parents.add(curr.get());
+      curr = curr.get().parent;
+    }
+    return ImmutableList.copyOf(parents);
+  }
+
+  public ImmutableList<String> getParentNames() {
+    return tree().stream().skip(1).map(p -> p.getProject().getName()).collect(toImmutableList());
   }
 }

java/com/google/gerrit/index/project/ProjectField.java

@@ -14,17 +14,24 @@
 
 package com.google.gerrit.index.project;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.gerrit.index.FieldDef.exact;
 import static com.google.gerrit.index.FieldDef.fullText;
 import static com.google.gerrit.index.FieldDef.prefix;
+import static com.google.gerrit.index.FieldDef.storedOnly;
 
-import com.google.common.collect.Iterables;
 import com.google.gerrit.index.FieldDef;
+import com.google.gerrit.index.RefState;
 import com.google.gerrit.index.SchemaUtil;
 import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.reviewdb.client.RefNames;
 
 /** Index schema for projects. */
 public class ProjectField {
+  private static byte[] toRefState(Project project) {
+    return RefState.create(RefNames.REFS_CONFIG, project.getConfigRefState())
+        .toByteArray(project.getNameKey());
+  }
 
   public static final FieldDef<ProjectData, String> NAME =
       exact("name").stored().build(p -> p.getProject().getName());
@@ -39,6 +46,22 @@ public class ProjectField {
       prefix("name_part").buildRepeatable(p -> SchemaUtil.getNameParts(p.getProject().getName()));
 
   public static final FieldDef<ProjectData, Iterable<String>> ANCESTOR_NAME =
-      exact("ancestor_name")
-          .buildRepeatable(p -> Iterables.transform(p.getAncestors(), Project.NameKey::get));
+      exact("ancestor_name").buildRepeatable(p -> p.getParentNames());
+
+  /**
+   * All values of all refs that were used in the course of indexing this document. This covers
+   * {@code refs/meta/config} of the current project and all of its parents.
+   *
+   * <p>Emitted as UTF-8 encoded strings of the form {@code project:ref/name:[hex sha]}.
+   */
+  public static final FieldDef<ProjectData, Iterable<byte[]>> REF_STATE =
+      storedOnly("ref_state")
+          .buildRepeatable(
+              projectData ->
+                  projectData
+                      .tree()
+                      .stream()
+                      .filter(p -> p.getProject().getConfigRefState() != null)
+                      .map(p -> toRefState(p.getProject()))
+                      .collect(toImmutableList()));
 }

java/com/google/gerrit/index/project/ProjectSchemaDefinitions.java

@@ -21,6 +21,7 @@ import com.google.gerrit.index.SchemaDefinitions;
 
 public class ProjectSchemaDefinitions extends SchemaDefinitions<ProjectData> {
 
+  @Deprecated
   static final Schema<ProjectData> V1 =
       schema(
           ProjectField.NAME,
@@ -29,6 +30,8 @@ public class ProjectSchemaDefinitions extends SchemaDefinitions<ProjectData> {
           ProjectField.NAME_PART,
           ProjectField.ANCESTOR_NAME);
 
+  static final Schema<ProjectData> V2 = schema(V1, ProjectField.REF_STATE);
+
   public static final ProjectSchemaDefinitions INSTANCE = new ProjectSchemaDefinitions();
 
   private ProjectSchemaDefinitions() {

java/com/google/gerrit/reviewdb/client/Project.java

@@ -99,6 +99,8 @@ public final class Project {
 
   protected String themeName;
 
+  protected String configRefState;
+
   protected Project() {}
 
   public Project(Project.NameKey nameKey) {
@@ -239,4 +241,14 @@ public final class Project {
   public void setParentName(NameKey n) {
     parent = n;
   }
+
+  /** Returns the {@code ObjectId} as 40 digit hex of {@code refs/meta/config}'s HEAD. */
+  public String getConfigRefState() {
+    return configRefState;
+  }
+
+  /** Sets the {@code ObjectId} as 40 digit hex of {@code refs/meta/config}'s HEAD. */
+  public void setConfigRefState(String state) {
+    configRefState = state;
+  }
 }

java/com/google/gerrit/server/index/account/AccountField.java

@@ -27,11 +27,11 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.gerrit.common.data.GlobalCapability;
 import com.google.gerrit.index.FieldDef;
+import com.google.gerrit.index.RefState;
 import com.google.gerrit.index.SchemaUtil;
 import com.google.gerrit.reviewdb.client.RefNames;
 import com.google.gerrit.server.account.AccountState;
 import com.google.gerrit.server.account.externalids.ExternalId;
-import com.google.gerrit.server.index.RefState;
 import java.sql.Timestamp;
 import java.util.Arrays;
 import java.util.Collections;

java/com/google/gerrit/server/index/account/StalenessChecker.java

@@ -24,6 +24,7 @@ import com.google.common.collect.ListMultimap;
 import com.google.common.collect.MultimapBuilder;
 import com.google.gerrit.index.IndexConfig;
 import com.google.gerrit.index.QueryOptions;
+import com.google.gerrit.index.RefState;
 import com.google.gerrit.index.query.FieldBundle;
 import com.google.gerrit.reviewdb.client.Account;
 import com.google.gerrit.reviewdb.client.Project;
@@ -33,7 +34,6 @@ import com.google.gerrit.server.account.externalids.ExternalIds;
 import com.google.gerrit.server.config.AllUsersName;
 import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.index.IndexUtils;
-import com.google.gerrit.server.index.RefState;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import java.io.IOException;

java/com/google/gerrit/server/index/change/ChangeField.java

@@ -43,6 +43,7 @@ import com.google.common.primitives.Longs;
 import com.google.gerrit.common.data.SubmitRecord;
 import com.google.gerrit.common.data.SubmitRequirement;
 import com.google.gerrit.index.FieldDef;
+import com.google.gerrit.index.RefState;
 import com.google.gerrit.index.SchemaUtil;
 import com.google.gerrit.mail.Address;
 import com.google.gerrit.reviewdb.client.Account;
@@ -57,7 +58,6 @@ import com.google.gerrit.server.ReviewerByEmailSet;
 import com.google.gerrit.server.ReviewerSet;
 import com.google.gerrit.server.StarredChangesUtil;
 import com.google.gerrit.server.config.AllUsersName;
-import com.google.gerrit.server.index.RefState;
 import com.google.gerrit.server.index.change.StalenessChecker.RefStatePattern;
 import com.google.gerrit.server.notedb.ChangeNotes;
 import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;

java/com/google/gerrit/server/index/change/StalenessChecker.java

@@ -31,12 +31,12 @@ import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.common.Nullable;
 import com.google.gerrit.index.IndexConfig;
+import com.google.gerrit.index.RefState;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.reviewdb.client.Project;
 import com.google.gerrit.reviewdb.server.ReviewDb;
 import com.google.gerrit.server.UsedAt;
 import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.index.RefState;
 import com.google.gerrit.server.notedb.ChangeNotes;
 import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
 import com.google.gerrit.server.query.change.ChangeData;

java/com/google/gerrit/server/project/ProjectConfig.java

@@ -499,6 +499,9 @@ public class ProjectConfig extends VersionedMetaData implements ValidationError.
     if (p.getDescription() == null) {
       p.setDescription("");
     }
+    if (revision != null) {
+      p.setConfigRefState(revision.toObjectId().name());
+    }
 
     if (rc.getStringList(ACCESS, null, KEY_INHERIT_FROM).length > 1) {
       // The config must not contain more than one parent to inherit from

java/com/google/gerrit/server/project/ProjectState.java

@@ -62,6 +62,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.lib.Ref;
@@ -538,7 +539,11 @@ public class ProjectState {
   }
 
   public ProjectData toProjectData() {
-    return new ProjectData(getProject(), parents().transform(s -> s.getProject().getNameKey()));
+    ProjectData project = null;
+    for (ProjectState state : treeInOrder()) {
+      project = new ProjectData(state.getProject(), Optional.ofNullable(project));
+    }
+    return project;
   }
 
   private String readFile(Path p) throws IOException {

javatests/com/google/gerrit/acceptance/api/project/BUILD

@@ -4,4 +4,5 @@ acceptance_tests(
     srcs = glob(["*IT.java"]),
     group = "api_project",
     labels = ["api"],
+    deps = ["//java/com/google/gerrit/index/project"],
 )

javatests/com/google/gerrit/acceptance/api/project/ProjectIndexerIT.java

@@ -0,0 +1,75 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.acceptance.api.project;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.acceptance.GitUtil.fetch;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.gerrit.acceptance.AbstractDaemonTest;
+import com.google.gerrit.index.IndexConfig;
+import com.google.gerrit.index.QueryOptions;
+import com.google.gerrit.index.RefState;
+import com.google.gerrit.index.project.ProjectField;
+import com.google.gerrit.index.project.ProjectIndex;
+import com.google.gerrit.index.project.ProjectIndexCollection;
+import com.google.gerrit.index.project.ProjectIndexer;
+import com.google.gerrit.index.query.FieldBundle;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.Inject;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Optional;
+import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.Ref;
+import org.junit.Test;
+
+public class ProjectIndexerIT extends AbstractDaemonTest {
+  @Inject private ProjectIndexer projectIndexer;
+  @Inject private ProjectIndexCollection indexes;
+  @Inject private IndexConfig indexConfig;
+
+  private static final ImmutableSet<String> FIELDS =
+      ImmutableSet.of(ProjectField.NAME.getName(), ProjectField.REF_STATE.getName());
+
+  @Test
+  public void indexProject_indexesRefStateOfProjectAndParents() throws Exception {
+    projectIndexer.index(project);
+    ProjectIndex i = indexes.getSearchIndex();
+    assertThat(i.getSchema().hasField(ProjectField.REF_STATE)).isTrue();
+
+    Optional<FieldBundle> result =
+        i.getRaw(project, QueryOptions.create(indexConfig, 0, 1, FIELDS));
+
+    assertThat(result.isPresent()).isTrue();
+    Iterable<byte[]> refState = result.get().getValue(ProjectField.REF_STATE);
+    assertThat(refState).isNotEmpty();
+
+    Map<Project.NameKey, Collection<RefState>> states = RefState.parseStates(refState).asMap();
+
+    fetch(testRepo, "refs/meta/config:refs/meta/config");
+    Ref projectConfigRef = testRepo.getRepository().exactRef("refs/meta/config");
+    TestRepository<InMemoryRepository> allProjectsRepo = cloneProject(allProjects, admin);
+    fetch(allProjectsRepo, "refs/meta/config:refs/meta/config");
+    Ref allProjectConfigRef = allProjectsRepo.getRepository().exactRef("refs/meta/config");
+    assertThat(states)
+        .containsExactly(
+            project,
+            ImmutableSet.of(RefState.of(projectConfigRef)),
+            allProjects,
+            ImmutableSet.of(RefState.of(allProjectConfigRef)));
+  }
+}

javatests/com/google/gerrit/server/index/change/StalenessCheckerTest.java

@@ -24,11 +24,11 @@ import static java.util.stream.Collectors.toList;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableSetMultimap;
 import com.google.common.collect.ListMultimap;
+import com.google.gerrit.index.RefState;
 import com.google.gerrit.reviewdb.client.Account;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.reviewdb.client.Project;
 import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.index.RefState;
 import com.google.gerrit.server.index.change.StalenessChecker.RefStatePattern;
 import com.google.gerrit.server.notedb.NoteDbChangeState;
 import com.google.gerrit.testing.GerritBaseTests;