Merge branch 'stable-3.0' into stable-3.1

* stable-3.0:
  Simplify Init for Elasticsearch
  Refactor reindexProjects in Init to be general
  Avoid auto-reindex of projects during init when unneeded

Change-Id: Id945a48cebe8408e86e3c4025fc6fbb1cb09bdc6

java/com/google/gerrit/pgm/Init.java

@@ -22,6 +22,7 @@ import com.google.common.collect.Sets;
 import com.google.gerrit.common.IoUtil;
 import com.google.gerrit.common.PageLinks;
 import com.google.gerrit.common.PluginData;
+import com.google.gerrit.index.SchemaDefinitions;
 import com.google.gerrit.index.project.ProjectSchemaDefinitions;
 import com.google.gerrit.pgm.init.BaseInit;
 import com.google.gerrit.pgm.init.Browser;
@@ -30,6 +31,10 @@ import com.google.gerrit.pgm.init.api.ConsoleUI;
 import com.google.gerrit.pgm.util.ErrorLogFile;
 import com.google.gerrit.server.config.GerritServerConfigModule;
 import com.google.gerrit.server.config.SitePath;
+import com.google.gerrit.server.index.GerritIndexStatus;
+import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
+import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
+import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
 import com.google.gerrit.server.ioutil.HostPlatform;
 import com.google.gerrit.server.securestore.SecureStoreClassName;
 import com.google.gerrit.server.util.ReplicaUtil;
@@ -60,9 +65,6 @@ public class Init extends BaseInit {
   @Option(name = "--no-auto-start", usage = "Don't automatically start daemon after init")
   private boolean noAutoStart;
 
-  @Option(name = "--no-reindex", usage = "Don't automatically reindex any entities")
-  private boolean noReindex;
-
   @Option(name = "--skip-plugins", usage = "Don't install plugins")
   private boolean skipPlugins;
 
@@ -91,6 +93,8 @@ public class Init extends BaseInit {
 
   @Inject Browser browser;
 
+  private GerritIndexStatus indexStatus;
+
   public Init() {
     super(new WarDistribution(), null);
   }
@@ -103,6 +107,7 @@ public class Init extends BaseInit {
 
   @Override
   protected boolean beforeInit(SiteInit init) throws Exception {
+    indexStatus = new GerritIndexStatus(init.site);
     ErrorLogFile.errorOnlyConsole();
 
     if (!skipPlugins) {
@@ -131,6 +136,12 @@ public class Init extends BaseInit {
 
   @Override
   protected void afterInit(SiteRun run) throws Exception {
+    List<SchemaDefinitions<?>> schemaDefs =
+        ImmutableList.of(
+            AccountSchemaDefinitions.INSTANCE,
+            ChangeSchemaDefinitions.INSTANCE,
+            GroupSchemaDefinitions.INSTANCE,
+            ProjectSchemaDefinitions.INSTANCE);
     List<Module> modules = new ArrayList<>();
     modules.add(
         new AbstractModule() {
@@ -146,7 +157,11 @@ public class Init extends BaseInit {
     modules.add(new GerritServerConfigModule());
     Guice.createInjector(modules).injectMembers(this);
     if (!ReplicaUtil.isReplica(run.flags.cfg)) {
-      reindexProjects();
+      for (SchemaDefinitions<?> schemaDef : schemaDefs) {
+        if (!indexStatus.exists(schemaDef.getName())) {
+          reindex(schemaDef);
+        }
+      }
     }
     start(run);
   }
@@ -259,11 +274,7 @@ public class Init extends BaseInit {
     }
   }
 
-  private void reindexProjects() throws Exception {
+  private void reindex(SchemaDefinitions<?> schemaDef) throws Exception {
-    if (noReindex) {
-      return;
-    }
-    // Reindex all projects, so that we bootstrap the project index for new installations
     List<String> reindexArgs =
         ImmutableList.of(
             "--site-path",
@@ -271,8 +282,9 @@ public class Init extends BaseInit {
             "--threads",
             Integer.toString(1),
             "--index",
-            ProjectSchemaDefinitions.NAME);
+            schemaDef.getName());
-    getConsoleUI().message("Init complete, reindexing projects with:");
+    getConsoleUI()
+        .message(String.format("Init complete, reindexing %s with:", schemaDef.getName()));
     getConsoleUI().message(" reindex " + reindexArgs.stream().collect(joining(" ")));
     Reindex reindexPgm = new Reindex();
     reindexPgm.main(reindexArgs.stream().toArray(String[]::new));

java/com/google/gerrit/server/index/GerritIndexStatus.java

@@ -44,6 +44,10 @@ public class GerritIndexStatus {
     return cfg.getBoolean(SECTION, indexDirName(indexName, version), KEY_READY, false);
   }
 
+  public boolean exists(String indexName) {
+    return cfg.getSubsections(SECTION).stream().anyMatch(n -> n.startsWith(indexName));
+  }
+
   public void save() throws IOException {
     cfg.save();
   }

javatests/com/google/gerrit/acceptance/pgm/ElasticReindexIT.java

@@ -21,7 +21,6 @@ import com.google.gerrit.elasticsearch.ElasticVersion;
 import com.google.gerrit.testing.ConfigSuite;
 import com.google.inject.Injector;
 import org.eclipse.jgit.lib.Config;
-import org.junit.Before;
 
 public class ElasticReindexIT extends AbstractReindexTests {
 
@@ -39,10 +38,4 @@ public class ElasticReindexIT extends AbstractReindexTests {
   public void configureIndex(Injector injector) {
     createAllIndexes(injector);
   }
-
-  @Before
-  public void reindexFirstSinceElastic() throws Exception {
-    assertServerStartupFails();
-    runGerrit("reindex", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
-  }
 }

javatests/com/google/gerrit/acceptance/pgm/InitIT.java

@@ -26,7 +26,16 @@ import com.google.gerrit.index.project.ProjectData;
 import com.google.gerrit.index.project.ProjectIndexCollection;
 import com.google.gerrit.server.config.AllProjectsName;
 import com.google.gerrit.server.config.AllUsersName;
+import java.io.IOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Comparator;
 import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.eclipse.jgit.util.FS;
 import org.junit.Test;
 
 @NoHttpd
@@ -34,7 +43,7 @@ public class InitIT extends StandaloneSiteTest {
 
   @Test
   public void indexesAllProjectsAndAllUsers() throws Exception {
-    runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+    initSite();
     try (ServerContext ctx = startServer()) {
       ProjectIndexCollection projectIndex =
           ctx.getInjector().getInstance(ProjectIndexCollection.class);
@@ -48,4 +57,52 @@ public class InitIT extends StandaloneSiteTest {
       assertThat(allUsersData).isPresent();
     }
   }
+
+  @Test
+  public void initDoesNotReindexProjectsOnExistingSites() throws Exception {
+    initSite();
+
+    // Simulate a projects indexes files modified in the past by 3 seconds
+    Optional<Instant> projectsLastModified =
+        getProjectsIndexLastModified(sitePaths.index_dir).map(t -> t.minusSeconds(3));
+    assertThat(projectsLastModified).isPresent();
+    setProjectsIndexLastModifiedInThePast(sitePaths.index_dir, projectsLastModified.get());
+
+    initSite();
+    Optional<Instant> projectsLastModifiedAfterInit =
+        getProjectsIndexLastModified(sitePaths.index_dir);
+
+    // Verify that projects index files haven't been updated
+    assertThat(projectsLastModified).isEqualTo(projectsLastModifiedAfterInit);
+  }
+
+  private void initSite() throws Exception {
+    runGerrit("init", "-d", sitePaths.site_path.toString(), "--show-stack-trace");
+  }
+
+  private void setProjectsIndexLastModifiedInThePast(Path indexDir, Instant time)
+      throws IOException {
+    for (Path path : getAllProjectsIndexFiles(indexDir).collect(Collectors.toList())) {
+      FS.DETECTED.setLastModified(path, time);
+    }
+  }
+
+  private Optional<Instant> getProjectsIndexLastModified(Path indexDir) throws IOException {
+    return getAllProjectsIndexFiles(indexDir)
+        .map(FS.DETECTED::lastModifiedInstant)
+        .max(Comparator.comparingLong(Instant::toEpochMilli));
+  }
+
+  private Stream<Path> getAllProjectsIndexFiles(Path indexDir) throws IOException {
+    Optional<Path> projectsPath =
+        Files.walk(indexDir, 1)
+            .filter(Files::isDirectory)
+            .filter(p -> p.getFileName().toString().startsWith("projects_"))
+            .findFirst();
+    if (!projectsPath.isPresent()) {
+      return Stream.empty();
+    }
+
+    return Files.walk(projectsPath.get(), 1, FileVisitOption.FOLLOW_LINKS);
+  }
 }