Scan index directory to determine Lucene versions

Mark index versions as "ready" when they are fully indexed and the
server is not running (i.e. from Reindex). By default, search from the
most recent ready index version, and write to both the most recent
ready version and (if different) the most recent known version.

At server startup, mark all versions except those we are about to
start writing to as not ready.

Change-Id: Icf42a3eb27b0445899300d60941cd701a8072d41

gerrit-lucene/BUCK

@@ -27,7 +27,9 @@ java_library(
     '//gerrit-server:server',
     '//lib:guava',
     '//lib:gwtorm',
+    '//lib:jsr305',
     '//lib/guice:guice',
+    '//lib/guice:guice-assistedinject',
     '//lib/jgit:jgit',
     '//lib/log:api',
     '//lib/lucene:analyzers-common',

gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java

@@ -1,92 +0,0 @@
-// Copyright (C) 2013 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.lucene;
-
-import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION;
-import static org.apache.lucene.util.Version.LUCENE_CURRENT;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.gerrit.extensions.events.LifecycleListener;
-import com.google.gerrit.server.config.SitePaths;
-import com.google.gerrit.server.index.ChangeSchemas;
-import com.google.inject.Inject;
-import com.google.inject.ProvisionException;
-
-import org.apache.lucene.util.Version;
-import org.eclipse.jgit.errors.ConfigInvalidException;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.util.FS;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-
-public class IndexVersionCheck implements LifecycleListener {
-  public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
-      LuceneChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(),
-      LuceneChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion());
-
-  public static File gerritIndexConfig(SitePaths sitePaths) {
-    return new File(sitePaths.index_dir, "gerrit_index.config");
-  }
-
-  private final SitePaths sitePaths;
-
-  @Inject
-  IndexVersionCheck(SitePaths sitePaths) {
-    this.sitePaths = sitePaths;
-  }
-
-  @Override
-  public void start() {
-    File file = gerritIndexConfig(sitePaths);
-    try {
-      FileBasedConfig cfg = new FileBasedConfig(file, FS.detect());
-      cfg.load();
-      for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
-        int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0);
-        if (schemaVersion != e.getValue()) {
-          throw new ProvisionException(String.format(
-              "wrong index schema version for \"%s\": expected %d, found %d%s",
-              e.getKey(), e.getValue(), schemaVersion, upgrade()));
-        }
-      }
-      @SuppressWarnings("deprecation")
-      Version luceneVersion =
-          cfg.getEnum("lucene", null, "version", LUCENE_CURRENT);
-      if (luceneVersion != LUCENE_VERSION) {
-        throw new ProvisionException(String.format(
-            "wrong Lucene version: expected %d, found %d%s",
-            luceneVersion, LUCENE_VERSION, upgrade()));
-
-      }
-    } catch (IOException e) {
-      throw new ProvisionException("unable to read " + file);
-    } catch (ConfigInvalidException e) {
-      throw new ProvisionException("invalid config file " + file);
-    }
-  }
-
-  @Override
-  public void stop() {
-    // Do nothing.
-  }
-
-  private final String upgrade() {
-    return "\nRun reindex to rebuild the index:\n"
-      + "$ java -jar gerrit.war reindex -d "
-      + sitePaths.site_path.getAbsolutePath();
-  }
-}

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java

@@ -14,8 +14,6 @@
 
 package com.google.gerrit.lucene;
 
-import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS;
-import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig;
 import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES;
 import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES;
 
@@ -26,7 +24,6 @@ import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningScheduledExecutorService;
-import com.google.gerrit.extensions.events.LifecycleListener;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.SitePaths;
@@ -35,7 +32,6 @@ import com.google.gerrit.server.index.ChangeIndex;
 import com.google.gerrit.server.index.FieldDef;
 import com.google.gerrit.server.index.FieldDef.FillArgs;
 import com.google.gerrit.server.index.FieldType;
-import com.google.gerrit.server.index.IndexCollection;
 import com.google.gerrit.server.index.IndexExecutor;
 import com.google.gerrit.server.index.IndexRewriteImpl;
 import com.google.gerrit.server.index.Schema;
@@ -45,6 +41,8 @@ import com.google.gerrit.server.query.change.ChangeData;
 import com.google.gerrit.server.query.change.ChangeDataSource;
 import com.google.gwtorm.server.OrmException;
 import com.google.gwtorm.server.ResultSet;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
@@ -69,7 +67,6 @@ import org.apache.lucene.util.Version;
 import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.util.FS;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -79,11 +76,12 @@ import java.sql.Timestamp;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
+import javax.annotation.Nullable;
+
 /**
  * Secondary index implementation using Apache Lucene.
  * <p>
@@ -92,7 +90,7 @@ import java.util.concurrent.Future;
  * though there may be some lag between a committed write and it showing up to
  * other threads' searchers.
  */
-public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
+public class LuceneChangeIndex implements ChangeIndex {
   private static final Logger log =
       LoggerFactory.getLogger(LuceneChangeIndex.class);
 
@@ -101,6 +99,10 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
   public static final String CHANGES_CLOSED = "closed";
   private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
 
+  static interface Factory {
+    LuceneChangeIndex create(Schema<ChangeData> schema, String base);
+  }
+
   private static IndexWriterConfig getIndexWriterConfig(Config cfg, String name) {
     IndexWriterConfig writerConfig = new IndexWriterConfig(LUCENE_VERSION,
         new StandardAnalyzer(LUCENE_VERSION));
@@ -115,28 +117,27 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
 
   private final SitePaths sitePaths;
   private final FillArgs fillArgs;
-  private final IndexCollection indexes;
   private final ExecutorService executor;
+  private final File dir;
   private final Schema<ChangeData> schema;
   private final SubIndex openIndex;
   private final SubIndex closedIndex;
 
-  LuceneChangeIndex(@GerritServerConfig Config cfg,
+  @AssistedInject
+  LuceneChangeIndex(
+      @GerritServerConfig Config cfg,
       SitePaths sitePaths,
-      IndexCollection indexes,
       @IndexExecutor ListeningScheduledExecutorService executor,
       FillArgs fillArgs,
-      Schema<ChangeData> schema,
+      @Assisted Schema<ChangeData> schema,
-      String base) throws IOException {
+      @Assisted @Nullable String base) throws IOException {
-    this.indexes = indexes;
     this.sitePaths = sitePaths;
     this.fillArgs = fillArgs;
     this.executor = executor;
     this.schema = schema;
 
-    File dir;
     if (base == null) {
-      dir = new File(sitePaths.index_dir, "changes_" + schema.getVersion());
+      dir = LuceneVersionManager.getDir(sitePaths, schema);
     } else {
       dir = new File(base);
     }
@@ -147,13 +148,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
   }
 
   @Override
-  public void start() {
+  public void close() {
-    indexes.setSearchIndex(this);
-    indexes.addWriteIndex(this);
-  }
-
-  @Override
-  public void stop() {
     List<Future<?>> closeFutures = Lists.newArrayListWithCapacity(2);
     closeFutures.add(executor.submit(new Runnable() {
       @Override
@@ -249,6 +244,18 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
     return new QuerySource(indexes, QueryBuilder.toQuery(p));
   }
 
+  @Override
+  public void markReady() throws IOException {
+    try {
+      FileBasedConfig cfg = LuceneVersionManager.loadGerritIndexConfig(sitePaths);
+      cfg.setBoolean("index", Integer.toString(schema.getVersion()), "ready",
+          true);
+      cfg.save();
+    } catch (ConfigInvalidException e) {
+      throw new IOException(e);
+    }
+  }
+
   private static class QuerySource implements ChangeDataSource {
     // TODO(dborowitz): Push limit down from predicate tree.
     private static final int LIMIT = 1000;
@@ -388,17 +395,4 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
   private static Field.Store store(FieldDef<?, ?> f) {
     return f.isStored() ? Field.Store.YES : Field.Store.NO;
   }
-
-  @Override
-  public void finishIndex() throws IOException,
-      ConfigInvalidException {
-    FileBasedConfig cfg =
-        new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
-
-    for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
-      cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
-    }
-    cfg.setEnum("lucene", null, "version", LUCENE_VERSION);
-    cfg.save();
-  }
 }

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java

@@ -14,57 +14,89 @@
 
 package com.google.gerrit.lucene;
 
-import com.google.common.util.concurrent.ListeningScheduledExecutorService;
+import com.google.gerrit.extensions.events.LifecycleListener;
 import com.google.gerrit.lifecycle.LifecycleModule;
-import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.FactoryModule;
 import com.google.gerrit.server.config.SitePaths;
-import com.google.gerrit.server.index.ChangeIndex;
 import com.google.gerrit.server.index.ChangeSchemas;
-import com.google.gerrit.server.index.FieldDef.FillArgs;
 import com.google.gerrit.server.index.IndexCollection;
-import com.google.gerrit.server.index.IndexExecutor;
 import com.google.gerrit.server.index.IndexModule;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.query.change.ChangeData;
+import com.google.inject.Inject;
 import com.google.inject.Provides;
 import com.google.inject.Singleton;
 
-import org.eclipse.jgit.lib.Config;
-
-import java.io.IOException;
-
 public class LuceneIndexModule extends LifecycleModule {
-  private final boolean checkVersion;
+  private final Integer singleVersion;
   private final int threads;
   private final String base;
 
   public LuceneIndexModule() {
-    this(true, 0, null);
+    this(null, 0, null);
   }
 
-  public LuceneIndexModule(boolean checkVersion, int threads,
+  public LuceneIndexModule(Integer singleVersion, int threads,
       String base) {
-    this.checkVersion = checkVersion;
+    this.singleVersion = singleVersion;
     this.threads = threads;
     this.base = base;
   }
 
   @Override
   protected void configure() {
-    install(new IndexModule(threads));
+    install(new FactoryModule() {
-    bind(ChangeIndex.class).to(LuceneChangeIndex.class);
+      @Override
-    listener().to(LuceneChangeIndex.class);
+      public void configure() {
-    if (checkVersion) {
+        factory(LuceneChangeIndex.Factory.class);
-      listener().to(IndexVersionCheck.class);
       }
+    });
+    install(new IndexModule(threads));
+    if (singleVersion == null && base == null) {
+      listener().to(LuceneVersionManager.class);
+    } else {
+      install(new SingleVersionModule());
+    }
+  }
+
+  private class SingleVersionModule extends LifecycleModule {
+    @Override
+    public void configure() {
+      listener().to(SingleVersionListener.class);
     }
 
     @Provides
     @Singleton
-  public LuceneChangeIndex getChangeIndex(@GerritServerConfig Config cfg,
+    LuceneChangeIndex getIndex(LuceneChangeIndex.Factory factory,
-      SitePaths sitePaths,
+        SitePaths sitePaths) {
-      IndexCollection indexes,
+      Schema<ChangeData> schema = singleVersion != null
-      @IndexExecutor ListeningScheduledExecutorService executor,
+          ? ChangeSchemas.get(singleVersion)
-      FillArgs fillArgs) throws IOException {
+          : ChangeSchemas.getLatest();
-    return new LuceneChangeIndex(cfg, sitePaths, indexes, executor, fillArgs,
+      return factory.create(schema, base);
-        ChangeSchemas.getLatestRelease(), base);
+    }
+  }
+
+  @Singleton
+  static class SingleVersionListener implements LifecycleListener {
+    private final IndexCollection indexes;
+    private final LuceneChangeIndex index;
+
+    @Inject
+    SingleVersionListener(IndexCollection indexes,
+        LuceneChangeIndex index) {
+      this.indexes = indexes;
+      this.index = index;
+    }
+
+    @Override
+    public void start() {
+      indexes.setSearchIndex(index);
+      indexes.addWriteIndex(index);
+    }
+
+    @Override
+    public void stop() {
+      index.close();
+    }
   }
 }

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java

@@ -0,0 +1,209 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package com.google.gerrit.server.git;
+
+package com.google.gerrit.lucene;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.primitives.Ints;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.ChangeSchemas;
+import com.google.gerrit.server.index.IndexCollection;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.query.change.ChangeData;
+import com.google.inject.Inject;
+import com.google.inject.ProvisionException;
+import com.google.inject.Singleton;
+
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.TreeMap;
+
+@Singleton
+class LuceneVersionManager implements LifecycleListener {
+  private static final Logger log = LoggerFactory
+      .getLogger(LuceneVersionManager.class);
+
+  private static final String CHANGES_PREFIX = "changes_";
+
+  private static class Version {
+    private final Schema<ChangeData> schema;
+    private final int version;
+    private final boolean exists;
+    private final boolean ready;
+
+    private Version(Schema<ChangeData> schema, int version, boolean exists,
+        boolean ready) {
+      checkArgument(schema == null || schema.getVersion() == version);
+      this.schema = schema;
+      this.version = version;
+      this.exists = exists;
+      this.ready = ready;
+    }
+  }
+
+  static File getDir(SitePaths sitePaths, Schema<ChangeData> schema) {
+    return new File(sitePaths.index_dir, String.format("%s%04d",
+        CHANGES_PREFIX, schema.getVersion()));
+  }
+
+  static FileBasedConfig loadGerritIndexConfig(SitePaths sitePaths)
+      throws ConfigInvalidException, IOException {
+    FileBasedConfig cfg = new FileBasedConfig(
+        new File(sitePaths.index_dir, "gerrit_index.config"), FS.detect());
+    cfg.load();
+    return cfg;
+  }
+
+  private static boolean getReady(Config cfg, int version) {
+    return cfg.getBoolean("index", Integer.toString(version), "ready", false);
+  }
+
+  private static void setReady(Config cfg, int version, boolean ready) {
+    cfg.setBoolean("index", Integer.toString(version), "ready", ready);
+  }
+
+  private final SitePaths sitePaths;
+  private final LuceneChangeIndex.Factory indexFactory;
+  private final IndexCollection indexes;
+
+  @Inject
+  LuceneVersionManager(
+      SitePaths sitePaths,
+      LuceneChangeIndex.Factory indexFactory,
+      IndexCollection indexes) {
+    this.sitePaths = sitePaths;
+    this.indexFactory = indexFactory;
+    this.indexes = indexes;
+  }
+
+  @Override
+  public void start() {
+    FileBasedConfig cfg;
+    try {
+      cfg = loadGerritIndexConfig(sitePaths);
+    } catch (ConfigInvalidException e) {
+      throw fail(e);
+    } catch (IOException e) {
+      throw fail(e);
+    }
+
+    TreeMap<Integer, Version> versions = scanVersions(cfg);
+    // Search from the most recent ready version.
+    // Write to the most recent ready version and the most recent version.
+    Version search = null;
+    List<Version> write = Lists.newArrayListWithCapacity(2);
+    for (Version v : versions.descendingMap().values()) {
+      if (v.schema == null) {
+        continue;
+      }
+      if (write.isEmpty()) {
+        write.add(v);
+      }
+      if (v.ready) {
+        search = v;
+        if (!write.contains(v)) {
+          write.add(v);
+        }
+        break;
+      }
+    }
+    if (search == null) {
+      throw new ProvisionException("No index versions ready; run Reindex");
+    }
+
+    markNotReady(cfg, versions.values(), write);
+    LuceneChangeIndex searchIndex = indexFactory.create(search.schema, null);
+    indexes.setSearchIndex(searchIndex);
+    for (Version v : write) {
+      if (v.schema != null) {
+        if (v.version != search.version) {
+          indexes.addWriteIndex(indexFactory.create(v.schema, null));
+        } else {
+          indexes.addWriteIndex(searchIndex);
+        }
+      }
+    }
+  }
+
+  private TreeMap<Integer, Version> scanVersions(Config cfg) {
+    TreeMap<Integer, Version> versions = Maps.newTreeMap();
+    for (Schema<ChangeData> schema : ChangeSchemas.ALL.values()) {
+      File f = getDir(sitePaths, schema);
+      boolean exists = f.exists() && f.isDirectory();
+      if (exists && !f.isDirectory()) {
+        log.warn("Not a directory: %s", f.getAbsolutePath());
+      }
+      int v = schema.getVersion();
+      versions.put(v, new Version(schema, v, exists, getReady(cfg, v)));
+    }
+
+    for (File f : sitePaths.index_dir.listFiles()) {
+      if (!f.getName().startsWith(CHANGES_PREFIX)) {
+        continue;
+      }
+      String versionStr = f.getName().substring(CHANGES_PREFIX.length());
+      Integer v = Ints.tryParse(versionStr);
+      if (v == null || versionStr.length() != 4) {
+        log.warn("Unrecognized version in index directory: {}",
+            f.getAbsolutePath());
+        continue;
+      }
+      if (!versions.containsKey(v)) {
+        versions.put(v, new Version(null, v, true, getReady(cfg, v)));
+      }
+    }
+    return versions;
+  }
+
+  private void markNotReady(FileBasedConfig cfg, Iterable<Version> versions,
+      Collection<Version> inUse) {
+    boolean dirty = false;
+    for (Version v : versions) {
+      if (!inUse.contains(v) && v.exists) {
+        setReady(cfg, v.version, false);
+        dirty = true;
+      }
+    }
+    if (dirty) {
+      try {
+        cfg.save();
+      } catch (IOException e) {
+        throw fail(e);
+      }
+    }
+  }
+
+  private ProvisionException fail(Throwable t) {
+    ProvisionException e = new ProvisionException("Error scanning indexes");
+    e.initCause(t);
+    throw e;
+  }
+
+  @Override
+  public void stop() {
+  }
+}

gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java

@@ -43,6 +43,8 @@ import com.google.gerrit.server.git.MultiProgressMonitor;
 import com.google.gerrit.server.git.MultiProgressMonitor.Task;
 import com.google.gerrit.server.index.ChangeIndex;
 import com.google.gerrit.server.index.ChangeIndexer;
+import com.google.gerrit.server.index.ChangeSchemas;
+import com.google.gerrit.server.index.IndexCollection;
 import com.google.gerrit.server.index.IndexExecutor;
 import com.google.gerrit.server.index.IndexModule;
 import com.google.gerrit.server.index.IndexModule.IndexType;
@@ -63,7 +65,6 @@ import com.google.inject.TypeLiteral;
 
 import org.eclipse.jgit.diff.DiffEntry;
 import org.eclipse.jgit.diff.DiffFormatter;
-import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectInserter;
@@ -98,14 +99,22 @@ public class Reindex extends SiteProgram {
   @Option(name = "--threads", usage = "Number of threads to use for indexing")
   private int threads = Runtime.getRuntime().availableProcessors();
 
+  @Option(name = "--schema-version",
+      usage = "Schema version to reindex; default is most recent version")
+  private Integer version;
+
   @Option(name = "--output", usage = "Prefix for output; path for local disk index, or prefix for remote index")
   private String outputBase;
 
   @Option(name = "--verbose", usage = "Output debug information for each change")
   private boolean verbose;
 
+  @Option(name = "--dry-run", usage = "Dry run: don't write anything to index")
+  private boolean dryRun;
+
   private Injector dbInjector;
   private Injector sysInjector;
+  private ChangeIndex index;
 
   @Override
   public int run() throws Exception {
@@ -114,22 +123,22 @@ public class Reindex extends SiteProgram {
     if (IndexModule.getIndexType(dbInjector) == IndexType.SQL) {
       throw die("index.type must be configured (or not SQL)");
     }
-
+    if (version == null) {
+      version = ChangeSchemas.getLatest().getVersion();
+    }
     LifecycleManager dbManager = new LifecycleManager();
     dbManager.add(dbInjector);
     dbManager.start();
 
     sysInjector = createSysInjector();
-
-    // Delete before any index may be created depending on this data.
-    deleteAll();
-
     LifecycleManager sysManager = new LifecycleManager();
     sysManager.add(sysInjector);
     sysManager.start();
 
+    index = sysInjector.getInstance(IndexCollection.class).getSearchIndex();
+    index.deleteAll();
     int result = indexAll();
-    writeVersion();
+    index.markReady();
 
     sysManager.stop();
     dbManager.stop();
@@ -142,7 +151,7 @@ public class Reindex extends SiteProgram {
     AbstractModule changeIndexModule;
     switch (IndexModule.getIndexType(dbInjector)) {
       case LUCENE:
-        changeIndexModule = new LuceneIndexModule(false, threads, outputBase);
+        changeIndexModule = new LuceneIndexModule(version, threads, outputBase);
         break;
       case SOLR:
         changeIndexModule = new SolrIndexModule(false, threads, outputBase);
@@ -207,11 +216,6 @@ public class Reindex extends SiteProgram {
     }
   }
 
-  private void deleteAll() throws IOException {
-    ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
-    index.deleteAll();
-  }
-
   private int indexAll() throws Exception {
     ReviewDb db = sysInjector.getInstance(ReviewDb.class);
     ListeningScheduledExecutorService executor = sysInjector.getInstance(
@@ -464,10 +468,4 @@ public class Reindex extends SiteProgram {
       }
     }
   }
-
-  private void writeVersion() throws IOException,
-      ConfigInvalidException {
-    ChangeIndex index = sysInjector.getInstance(ChangeIndex.class);
-    index.finishIndex();
-  }
 }

gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java

@@ -21,8 +21,6 @@ import com.google.gerrit.server.query.QueryParseException;
 import com.google.gerrit.server.query.change.ChangeData;
 import com.google.gerrit.server.query.change.ChangeDataSource;
 
-import org.eclipse.jgit.errors.ConfigInvalidException;
-
 import java.io.IOException;
 
 /**
@@ -64,20 +62,27 @@ public interface ChangeIndex {
     }
 
     @Override
-    public ChangeDataSource getSource(Predicate<ChangeData> p)
+    public ChangeDataSource getSource(Predicate<ChangeData> p) {
-        throws QueryParseException {
       throw new UnsupportedOperationException();
     }
 
     @Override
-    public void finishIndex() {
+    public void close() {
       // Do nothing.
     }
+
+    @Override
+    public void markReady() {
+      throw new UnsupportedOperationException();
+    }
   };
 
   /** @return the schema version used by this index. */
   public Schema<ChangeData> getSchema();
 
+  /** Close this index. */
+  public void close();
+
   /**
    * Insert a change document into the index.
    * <p>
@@ -137,11 +142,13 @@ public interface ChangeIndex {
       throws QueryParseException;
 
   /**
-   * Mark completion of indexing.
+   * Mark this index as up-to-date and ready to serve reads.
+   * <p>
+   * Should only be called immediately after a reindex, either during an online
+   * schema upgrade while actively writing to this index, or during an offline
+   * reindex.
    *
-   * @throws ConfigInvalidException
    * @throws IOException
    */
-  public void finishIndex() throws IOException,
+  public void markReady() throws IOException;
-      ConfigInvalidException;
 }

gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java

@@ -15,9 +15,9 @@
 package com.google.gerrit.server.index;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.gerrit.server.query.change.ChangeData;
 
@@ -58,23 +58,16 @@ public class ChangeSchemas {
     return new Schema<ChangeData>(false, Arrays.asList(fields));
   }
 
-  private static final ImmutableMap<Integer, Schema<ChangeData>> ALL;
+  public static final ImmutableMap<Integer, Schema<ChangeData>> ALL;
 
-  public Schema<ChangeData> get(int version) {
+  public static Schema<ChangeData> get(int version) {
     Schema<ChangeData> schema = ALL.get(version);
     checkArgument(schema != null, "Unrecognized schema version: %s", version);
     return schema;
   }
 
-  public static Schema<ChangeData> getLatestRelease() {
+  public static Schema<ChangeData> getLatest() {
-    Schema<ChangeData> latest = null;
+    return Iterables.getLast(ALL.values());
-    for (Schema<ChangeData> schema : ALL.values()) {
-      if (schema.isRelease()) {
-        latest = schema;
-      }
-    }
-    checkState(latest != null, "No released schema versions found");
-    return latest;
   }
 
   static {

gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java

@@ -16,6 +16,7 @@ package com.google.gerrit.server.index;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
+import com.google.gerrit.extensions.events.LifecycleListener;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 
@@ -28,7 +29,7 @@ import javax.annotation.Nullable;
 
 /** Dynamic pointers to the index versions used for searching and writing. */
 @Singleton
-public class IndexCollection {
+public class IndexCollection implements LifecycleListener {
   private final CopyOnWriteArrayList<ChangeIndex> writeIndexes;
   private final AtomicReference<ChangeIndex> searchIndex;
 
@@ -66,4 +67,21 @@ public class IndexCollection {
     }
     writeIndexes.add(index);
   }
+
+  @Override
+  public void start() {
+  }
+
+  @Override
+  public void stop() {
+    ChangeIndex read = searchIndex.get();
+    if (read != null) {
+      read.close();
+    }
+    for (ChangeIndex write : writeIndexes) {
+      if (write != read) {
+        write.close();
+      }
+    }
+  }
 }

gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java

@@ -16,11 +16,11 @@ package com.google.gerrit.server.index;
 
 import com.google.common.util.concurrent.ListeningScheduledExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
+import com.google.gerrit.lifecycle.LifecycleModule;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.git.WorkQueue;
 import com.google.gerrit.server.git.WorkQueue.Executor;
 import com.google.gerrit.server.query.change.ChangeQueryRewriter;
-import com.google.inject.AbstractModule;
 import com.google.inject.Injector;
 import com.google.inject.Key;
 import com.google.inject.Provides;
@@ -34,7 +34,7 @@ import org.eclipse.jgit.lib.Config;
  * This module should not be used directly except by specific secondary indexer
  * implementations (e.g. Lucene).
  */
-public class IndexModule extends AbstractModule {
+public class IndexModule extends LifecycleModule {
   public enum IndexType {
     SQL, LUCENE, SOLR;
   }
@@ -57,6 +57,8 @@ public class IndexModule extends AbstractModule {
     bind(ChangeIndexer.class).to(ChangeIndexerImpl.class);
     bind(ChangeQueryRewriter.class).to(IndexRewriteImpl.class);
     bind(IndexRewriteImpl.BasicRewritesImpl.class);
+    bind(IndexCollection.class);
+    listener().to(IndexCollection.class);
   }
 
   @Provides

gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java

@@ -90,7 +90,11 @@ public class IndexRewriteTest extends TestCase {
     }
 
     @Override
-    public void finishIndex() {
+    public void close() {
+    }
+
+    @Override
+    public void markReady() {
       throw new UnsupportedOperationException();
     }
   }

gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java

@@ -31,8 +31,8 @@ import java.util.Map;
 
 class IndexVersionCheck implements LifecycleListener {
   public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
-      SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(),
+      SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatest().getVersion(),
-      SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion());
+      SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatest().getVersion());
 
   public static File solrIndexConfig(SitePaths sitePaths) {
     return new File(sitePaths.index_dir, "gerrit_index.config");

gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java

@@ -52,7 +52,6 @@ import org.apache.solr.client.solrj.impl.CloudSolrServer;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrInputDocument;
-import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.eclipse.jgit.util.FS;
@@ -120,6 +119,11 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
     return schema;
   }
 
+  @Override
+  public void close() {
+    stop();
+  }
+
   @Override
   public ListenableFuture<Void> insert(ChangeData cd) throws IOException {
     String id = cd.getId().toString();
@@ -321,8 +325,7 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
   }
 
   @Override
-  public void finishIndex() throws IOException,
+  public void markReady() throws IOException {
-      ConfigInvalidException {
     // TODO Move the schema version information to a special meta-document
     FileBasedConfig cfg = new FileBasedConfig(
         solrIndexConfig(sitePaths),

gerrit-solr/src/main/java/com/google/gerrit/solr/SolrIndexModule.java

@@ -61,6 +61,6 @@ public class SolrIndexModule extends LifecycleModule {
       IndexCollection indexes,
       FillArgs fillArgs) throws IOException {
     return new SolrChangeIndex(cfg, fillArgs, sitePaths, indexes,
-        ChangeSchemas.getLatestRelease(), base);
+        ChangeSchemas.getLatest(), base);
   }
 }