Keep track of Lucene and index schema versions

Like SchemaVersionCheck, use a LifecycleManager to check the schema
versions on startup. We need to check both:

 - The on-disk format of the Lucene index. We specify the current
   version manually in LuceneChangeIndex, since Lucene itself doesn't
   store a version in the directory. (CheckIndex claims to be able to
   detect an out-of-date index file format, but it seems buggy.)
 - The version of the ChangeField schema, which indicates the
   semantics of the various fields.

Persist both of these versions in a simple config file,
index/gerrit_index.config.

If the check fails, inform the user they need to reindex. Rewrite
Reindex to delete all the files in the Lucene directory, not just the
documents.

Change-Id: Ib82b8aab21c819db3625cd9984262a40135e0894

gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java

@@ -0,0 +1,92 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package com.google.gerrit.server.git;
+
+package com.google.gerrit.lucene;
+
+import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION;
+
+import static org.apache.lucene.util.Version.LUCENE_CURRENT;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.ChangeField;
+import com.google.inject.Inject;
+import com.google.inject.ProvisionException;
+
+import org.apache.lucene.util.Version;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+
+public class IndexVersionCheck implements LifecycleListener {
+  public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
+      "changes", ChangeField.SCHEMA_VERSION);
+
+  public static File gerritIndexConfig(SitePaths sitePaths) {
+    return new File(sitePaths.index_dir, "gerrit_index.config");
+  }
+
+  private final SitePaths sitePaths;
+
+  @Inject
+  IndexVersionCheck(SitePaths sitePaths) {
+    this.sitePaths = sitePaths;
+  }
+
+  @Override
+  public void start() {
+    File file = gerritIndexConfig(sitePaths);
+    try {
+      FileBasedConfig cfg = new FileBasedConfig(file, FS.detect());
+      cfg.load();
+      for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
+        int schemaVersion = cfg.getInt("index", e.getKey(), "schemaVersion", 0);
+        if (schemaVersion != e.getValue()) {
+          throw new ProvisionException(String.format(
+              "wrong index schema version for \"%s\": expected %d, found %d%s",
+              e.getKey(), e.getValue(), schemaVersion, upgrade()));
+        }
+      }
+      @SuppressWarnings("deprecation")
+      Version luceneVersion =
+          cfg.getEnum("lucene", null, "version", LUCENE_CURRENT);
+      if (luceneVersion != LUCENE_VERSION) {
+        throw new ProvisionException(String.format(
+            "wrong Lucene version: expected %d, found %d%s",
+            luceneVersion, LUCENE_VERSION, upgrade()));
+
+      }
+    } catch (IOException e) {
+      throw new ProvisionException("unable to read " + file);
+    } catch (ConfigInvalidException e) {
+      throw new ProvisionException("invalid config file " + file);
+    }
+  }
+
+  @Override
+  public void stop() {
+    // Do nothing.
+  }
+
+  private final String upgrade() {
+    return "\nRun reindex to rebuild the index:\n"
+      + "$ java -jar gerrit.war reindex -d "
+      + sitePaths.site_path.getAbsolutePath();
+  }
+}

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java

@@ -75,7 +75,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
   private static final Logger log =
       LoggerFactory.getLogger(LuceneChangeIndex.class);
 
-  private static final Version VERSION = Version.LUCENE_43;
+  public static final Version LUCENE_VERSION = Version.LUCENE_43;
 
   private final FillArgs fillArgs;
   private final Directory dir;
@@ -88,7 +88,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
     this.fillArgs = fillArgs;
     dir = FSDirectory.open(new File(sitePaths.index_dir, "changes"));
     IndexWriterConfig writerConfig =
-        new IndexWriterConfig(VERSION, new StandardAnalyzer(VERSION));
+        new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION));
     writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
     writer = new IndexWriter(dir, writerConfig);
     searcherManager = new SearcherManager(writer, true, null);
@@ -143,6 +143,10 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
     }
   }
 
+  public Directory getDirectory() {
+    return dir;
+  }
+
   public IndexWriter getWriter() {
     return writer;
   }

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java

@@ -30,10 +30,23 @@ public class LuceneIndexModule extends LifecycleModule {
         .getBoolean("index", null, "enabled", false);
   }
 
+  private final boolean checkVersion;
+
+  public LuceneIndexModule() {
+    this(true);
+  }
+
+  public LuceneIndexModule(boolean checkVersion) {
+    this.checkVersion = checkVersion;
+  }
+
   @Override
   protected void configure() {
     bind(ChangeIndex.class).to(LuceneChangeIndex.class);
     bind(ChangeIndexer.class).to(ChangeIndexerImpl.class);
     listener().to(LuceneChangeIndex.class);
+    if (checkVersion) {
+      listener().to(IndexVersionCheck.class);
+    }
   }
 }

gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java

@@ -14,6 +14,9 @@
 
 package com.google.gerrit.pgm;
 
+import static com.google.gerrit.lucene.IndexVersionCheck.SCHEMA_VERSIONS;
+import static com.google.gerrit.lucene.IndexVersionCheck.gerritIndexConfig;
+import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION;
 import static com.google.gerrit.server.schema.DataSourceProvider.Context.SINGLE_USER;
 
 import com.google.common.base.Stopwatch;
@@ -40,10 +43,14 @@ import com.google.inject.TypeLiteral;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -86,6 +93,7 @@ public class Reindex extends SiteProgram {
     index.getWriter().commit();
     double elapsed = sw.elapsed(TimeUnit.MILLISECONDS) / 1000d;
     System.out.format("Reindexed %d changes in %.02fms", i, elapsed);
+    writeVersion();
 
     manager.stop();
     return 0;
@@ -94,7 +102,7 @@ public class Reindex extends SiteProgram {
   private Injector createSysInjector() {
     List<Module> modules = Lists.newArrayList();
     modules.add(PatchListCacheImpl.module());
-    modules.add(new LuceneIndexModule());
+    modules.add(new LuceneIndexModule(false));
     modules.add(new AbstractModule() {
       @SuppressWarnings("rawtypes")
       @Override
@@ -128,4 +136,16 @@ public class Reindex extends SiteProgram {
       }
     }
   }
+
+  private void writeVersion() throws IOException, ConfigInvalidException {
+    FileBasedConfig cfg =
+        new FileBasedConfig(gerritIndexConfig(sitePaths), FS.detect());
+    cfg.load();
+
+    for (Map.Entry<String, Integer> e : SCHEMA_VERSIONS.entrySet()) {
+      cfg.setInt("index", e.getKey(), "schemaVersion", e.getValue());
+    }
+    cfg.setEnum("lucene", null, "version", LUCENE_VERSION);
+    cfg.save();
+  }
 }

gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java

@@ -36,6 +36,9 @@ import java.util.Map;
  * Used to generate a schema for index implementations that require one.
  */
 public class ChangeField {
+  /** Increment whenever making schema changes. */
+  public static final int SCHEMA_VERSION = 1;
+
   /** Legacy change ID. */
   public static final FieldDef<ChangeData, Integer> CHANGE_ID =
       new FieldDef.Single<ChangeData, Integer>(ChangeQueryBuilder.FIELD_CHANGE,