Avoid auto-reindex of projects during init when unneeded

The forced reindex is not needed when the project index is present.

Preserve the automatic reindex of projects for new sites and defer
to reindex site program or online reindexing if project index already
exists.

The check is implemented in index backend agnostic manner, so that it
should work for both supported backends: Lucene and Elasticsearch. It
should also work independently whether or not a project index schema
migration is needed during upgrade.

Reindexing projects is a very expensive operation and can turn a simple
upgrade to a long and painful operation because of the increase of the
migration time and the amount of memory needed.

Bug: Issue 12680
Change-Id: I3a4f1d07405f7bb631467d1f005d48cb56ee867f

java/com/google/gerrit/pgm/Init.java

@@ -30,6 +30,7 @@ import com.google.gerrit.pgm.init.api.ConsoleUI;
 import com.google.gerrit.pgm.util.ErrorLogFile;
 import com.google.gerrit.server.config.GerritServerConfigModule;
 import com.google.gerrit.server.config.SitePath;
+import com.google.gerrit.server.index.GerritIndexStatus;
 import com.google.gerrit.server.ioutil.HostPlatform;
 import com.google.gerrit.server.securestore.SecureStoreClassName;
 import com.google.gerrit.server.util.ReplicaUtil;
@@ -60,9 +61,6 @@ public class Init extends BaseInit {
   @Option(name = "--no-auto-start", usage = "Don't automatically start daemon after init")
   private boolean noAutoStart;
 
-  @Option(name = "--no-reindex", usage = "Don't automatically reindex any entities")
-  private boolean noReindex;
-
   @Option(name = "--skip-plugins", usage = "Don't install plugins")
   private boolean skipPlugins;
 
@@ -91,6 +89,8 @@ public class Init extends BaseInit {
 
   @Inject Browser browser;
 
+  private boolean projectsIndexExists;
+
   public Init() {
     super(new WarDistribution(), null);
   }
@@ -103,6 +103,7 @@ public class Init extends BaseInit {
 
   @Override
   protected boolean beforeInit(SiteInit init) throws Exception {
+    projectsIndexExists = new GerritIndexStatus(init.site).exists(ProjectSchemaDefinitions.NAME);
     ErrorLogFile.errorOnlyConsole();
 
     if (!skipPlugins) {
@@ -145,7 +146,7 @@ public class Init extends BaseInit {
         });
     modules.add(new GerritServerConfigModule());
     Guice.createInjector(modules).injectMembers(this);
-    if (!ReplicaUtil.isReplica(run.flags.cfg)) {
+    if (!ReplicaUtil.isReplica(run.flags.cfg) && !projectsIndexExists) {
       reindexProjects();
     }
     start(run);
@@ -260,9 +261,6 @@ public class Init extends BaseInit {
   }
 
   private void reindexProjects() throws Exception {
-    if (noReindex) {
-      return;
-    }
     // Reindex all projects, so that we bootstrap the project index for new installations
     List<String> reindexArgs =
         ImmutableList.of(

java/com/google/gerrit/server/index/GerritIndexStatus.java

@@ -48,6 +48,10 @@ public class GerritIndexStatus {
     return cfg.getBoolean(SECTION, indexDirName(indexName, version), KEY_READY, false);
   }
 
+  public boolean exists(String indexName) {
+    return cfg.getSubsections(SECTION).stream().anyMatch(n -> n.startsWith(indexName));
+  }
+
   public void save() throws IOException {
     cfg.save();
   }

javatests/com/google/gerrit/acceptance/pgm/InitIT.java

@@ -26,7 +26,16 @@ import com.google.gerrit.index.project.ProjectData;
 import com.google.gerrit.index.project.ProjectIndexCollection;
 import com.google.gerrit.server.config.AllProjectsName;
 import com.google.gerrit.server.config.AllUsersName;
+import java.io.IOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Comparator;
 import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.eclipse.jgit.util.FS;
 import org.junit.Test;
 
 @NoHttpd
@@ -34,7 +43,7 @@ public class InitIT extends StandaloneSiteTest {
 
   @Test
   public void indexesAllProjectsAndAllUsers() throws Exception {
-    runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+    initSite();
     try (ServerContext ctx = startServer()) {
       ProjectIndexCollection projectIndex =
           ctx.getInjector().getInstance(ProjectIndexCollection.class);
@@ -48,4 +57,52 @@ public class InitIT extends StandaloneSiteTest {
       assertThat(allUsersData).isPresent();
     }
   }
+
+  @Test
+  public void initDoesNotReindexProjectsOnExistingSites() throws Exception {
+    initSite();
+
+    // Simulate a projects indexes files modified in the past by 3 seconds
+    Optional<Instant> projectsLastModified =
+        getProjectsIndexLastModified(sitePaths.index_dir).map(t -> t.minusSeconds(3));
+    assertThat(projectsLastModified).isPresent();
+    setProjectsIndexLastModifiedInThePast(sitePaths.index_dir, projectsLastModified.get());
+
+    initSite();
+    Optional<Instant> projectsLastModifiedAfterInit =
+        getProjectsIndexLastModified(sitePaths.index_dir);
+
+    // Verify that projects index files haven't been updated
+    assertThat(projectsLastModified).isEqualTo(projectsLastModifiedAfterInit);
+  }
+
+  private void initSite() throws Exception {
+    runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+  }
+
+  private void setProjectsIndexLastModifiedInThePast(Path indexDir, Instant time)
+      throws IOException {
+    for (Path path : getAllProjectsIndexFiles(indexDir).collect(Collectors.toList())) {
+      FS.DETECTED.setLastModified(path, time);
+    }
+  }
+
+  private Optional<Instant> getProjectsIndexLastModified(Path indexDir) throws IOException {
+    return getAllProjectsIndexFiles(indexDir)
+        .map(FS.DETECTED::lastModifiedInstant)
+        .max(Comparator.comparingLong(Instant::toEpochMilli));
+  }
+
+  private Stream<Path> getAllProjectsIndexFiles(Path indexDir) throws IOException {
+    Optional<Path> projectsPath =
+        Files.walk(indexDir, 1)
+            .filter(Files::isDirectory)
+            .filter(p -> p.getFileName().toString().startsWith("projects_"))
+            .findFirst();
+    if (!projectsPath.isPresent()) {
+      return Stream.empty();
+    }
+
+    return Files.walk(projectsPath.get(), 1, FileVisitOption.FOLLOW_LINKS);
+  }
 }