Merge changes from topic 'account-index-gh16'

* changes:
  Encapsulate per-index utilities in a pair of generic classes
  Generify OnlineReindexer and friends
  Move Document generation to AbstractLuceneIndex
  Parameterize Lucene's QueryBuilder
  Extract SubIndex as an abstract Index implementation
  LuceneVersionManager: Pass "changes_" prefix as argument
  Extract a method for in-memory testing state in Lucene
  Move BooleanQuery static call into LuceneChangeModule
  Move GerritIndexWriterConfig to its own file
  Schema for secondary index over accounts

gerrit-acceptance-framework/src/test/java/com/google/gerrit/acceptance/GerritServer.java

@@ -26,7 +26,6 @@ import com.google.gerrit.pgm.Daemon;
 import com.google.gerrit.pgm.Init;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.git.AsyncReceiveCommits;
-import com.google.gerrit.server.index.change.ChangeSchemas;
 import com.google.gerrit.server.ssh.NoSshModule;
 import com.google.gerrit.server.util.SocketUtil;
 import com.google.gerrit.server.util.SystemLog;
@@ -142,8 +141,7 @@ public class GerritServer {
       cfg.setBoolean("index", "lucene", "testInmemory", true);
       cfg.setString("gitweb", null, "cgi", "");
       daemon.setEnableHttpd(desc.httpd());
-      daemon.setLuceneModule(new LuceneIndexModule(
+      daemon.setLuceneModule(LuceneIndexModule.singleVersionAllLatest(0));
-          ChangeSchemas.getLatest().getVersion(), 0));
       daemon.setDatabaseForTesting(ImmutableList.<Module>of(
           new InMemoryTestingDatabaseModule(cfg)));
       daemon.start();

gerrit-lucene/src/main/java/com/google/gerrit/lucene/AbstractLuceneIndex.java

@@ -20,9 +20,22 @@ import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.AbstractFuture;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.gerrit.lucene.LuceneChangeIndex.GerritIndexWriterConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.FieldDef;
+import com.google.gerrit.server.index.FieldDef.FillArgs;
+import com.google.gerrit.server.index.FieldType;
+import com.google.gerrit.server.index.Index;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.index.Schema.Values;
 
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TrackingIndexWriter;
@@ -33,12 +46,13 @@ import org.apache.lucene.search.ReferenceManager.RefreshListener;
 import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.nio.file.Path;
+import java.sql.Timestamp;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
@@ -46,25 +60,45 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-/** Piece of the change index that is implemented as a separate Lucene index. */
+/** Basic Lucene index implementation. */
-public class SubIndex {
+public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
-  private static final Logger log = LoggerFactory.getLogger(SubIndex.class);
+  private static final Logger log =
+      LoggerFactory.getLogger(AbstractLuceneIndex.class);
 
+  static String sortFieldName(FieldDef<?, ?> f) {
+    return f.getName() + "_SORT";
+  }
+
+  public static void setReady(SitePaths sitePaths, int version, boolean ready)
+      throws IOException {
+    try {
+      // TODO(dborowitz): Totally broken for non-change indexes.
+      FileBasedConfig cfg =
+          LuceneVersionManager.loadGerritIndexConfig(sitePaths);
+      LuceneVersionManager.setReady(cfg, version, ready);
+      cfg.save();
+    } catch (ConfigInvalidException e) {
+      throw new IOException(e);
+    }
+  }
+
+  private final Schema<V> schema;
+  private final SitePaths sitePaths;
   private final Directory dir;
   private final TrackingIndexWriter writer;
   private final ReferenceManager<IndexSearcher> searcherManager;
   private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
   private final Set<NrtFuture> notDoneNrtFutures;
 
-  SubIndex(Path path, GerritIndexWriterConfig writerConfig,
+  AbstractLuceneIndex(
-      SearcherFactory searcherFactory) throws IOException {
+      Schema<V> schema,
-    this(FSDirectory.open(path), path.getFileName().toString(), writerConfig,
+      SitePaths sitePaths,
-        searcherFactory);
+      Directory dir,
-  }
+      final String name,
-
-  SubIndex(Directory dir, final String dirName,
       GerritIndexWriterConfig writerConfig,
       SearcherFactory searcherFactory) throws IOException {
+    this.schema = schema;
+    this.sitePaths = sitePaths;
     this.dir = dir;
     IndexWriter delegateWriter;
     long commitPeriod = writerConfig.getCommitWithinMs();
@@ -80,7 +114,7 @@ public class SubIndex {
       delegateWriter = autoCommitWriter;
 
       new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder()
-          .setNameFormat("Commit-%d " + dirName)
+          .setNameFormat("Commit-%d " + name)
           .setDaemon(true)
           .build())
           .scheduleAtFixedRate(new Runnable() {
@@ -92,14 +126,14 @@ public class SubIndex {
                   autoCommitWriter.commit();
                 }
               } catch (IOException e) {
-                log.error("Error committing Lucene index " + dirName, e);
+                log.error("Error committing " + name + " Lucene index", e);
               } catch (OutOfMemoryError e) {
-                log.error("Error committing Lucene index " + dirName, e);
+                log.error("Error committing " + name + " Lucene index", e);
                 try {
                   autoCommitWriter.close();
                 } catch (IOException e2) {
-                  log.error("SEVERE: Error closing Lucene index " + dirName
+                  log.error("SEVERE: Error closing " + name
-                      + " after OOM; index may be corrupted.", e);
+                      + " Lucene index  after OOM; index may be corrupted.", e);
                 }
               }
             }
@@ -115,7 +149,7 @@ public class SubIndex {
         writer, searcherManager,
         0.500 /* maximum stale age (seconds) */,
         0.010 /* minimum stale age (seconds) */);
-    reopenThread.setName("NRT " + dirName);
+    reopenThread.setName("NRT " + name);
     reopenThread.setPriority(Math.min(
         Thread.currentThread().getPriority() + 2,
         Thread.MAX_PRIORITY));
@@ -145,7 +179,13 @@ public class SubIndex {
     reopenThread.start();
   }
 
-  void close() {
+  @Override
+  public void markReady(boolean ready) throws IOException {
+    setReady(sitePaths, schema.getVersion(), ready);
+  }
+
+  @Override
+  public void close() {
     reopenThread.close();
 
     // Closing the reopen thread sets its generation to Long.MAX_VALUE, but we
@@ -187,7 +227,8 @@ public class SubIndex {
     return new NrtFuture(writer.deleteDocuments(term));
   }
 
-  void deleteAll() throws IOException {
+  @Override
+  public void deleteAll() throws IOException {
     writer.deleteAll();
   }
 
@@ -203,6 +244,55 @@ public class SubIndex {
     searcherManager.release(searcher);
   }
 
+  Document toDocument(V obj, FillArgs fillArgs) {
+    Document result = new Document();
+    for (Values<V> vs : schema.buildFields(obj, fillArgs)) {
+      if (vs.getValues() != null) {
+        add(result, vs);
+      }
+    }
+    return result;
+  }
+
+  void add(Document doc, Values<V> values) {
+    String name = values.getField().getName();
+    FieldType<?> type = values.getField().getType();
+    Store store = store(values.getField());
+
+    if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) {
+      for (Object value : values.getValues()) {
+        doc.add(new IntField(name, (Integer) value, store));
+      }
+    } else if (type == FieldType.LONG) {
+      for (Object value : values.getValues()) {
+        doc.add(new LongField(name, (Long) value, store));
+      }
+    } else if (type == FieldType.TIMESTAMP) {
+      for (Object value : values.getValues()) {
+        doc.add(new LongField(name, ((Timestamp) value).getTime(), store));
+      }
+    } else if (type == FieldType.EXACT
+        || type == FieldType.PREFIX) {
+      for (Object value : values.getValues()) {
+        doc.add(new StringField(name, (String) value, store));
+      }
+    } else if (type == FieldType.FULL_TEXT) {
+      for (Object value : values.getValues()) {
+        doc.add(new TextField(name, (String) value, store));
+      }
+    } else if (type == FieldType.STORED_ONLY) {
+      for (Object value : values.getValues()) {
+        doc.add(new StoredField(name, (byte[]) value));
+      }
+    } else {
+      throw FieldType.badFieldType(type);
+    }
+  }
+
+  private static Field.Store store(FieldDef<?, ?> f) {
+    return f.isStored() ? Field.Store.YES : Field.Store.NO;
+  }
+
   private final class NrtFuture extends AbstractFuture<Void> {
     private final long gen;
 
@@ -287,4 +377,9 @@ public class SubIndex {
       }
     }
   }
+
+  @Override
+  public Schema<V> getSchema() {
+    return schema;
+  }
 }

gerrit-lucene/src/main/java/com/google/gerrit/lucene/ChangeSubIndex.java

@@ -0,0 +1,98 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.lucene;
+
+import static com.google.common.collect.Iterables.getOnlyElement;
+import static com.google.gerrit.lucene.LuceneChangeIndex.ID_SORT_FIELD;
+import static com.google.gerrit.lucene.LuceneChangeIndex.UPDATED_SORT_FIELD;
+
+import com.google.gerrit.reviewdb.client.Change;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.FieldDef;
+import com.google.gerrit.server.index.QueryOptions;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.index.Schema.Values;
+import com.google.gerrit.server.index.change.ChangeField;
+import com.google.gerrit.server.index.change.ChangeIndex;
+import com.google.gerrit.server.query.DataSource;
+import com.google.gerrit.server.query.Predicate;
+import com.google.gerrit.server.query.QueryParseException;
+import com.google.gerrit.server.query.change.ChangeData;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.sql.Timestamp;
+
+public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData>
+    implements ChangeIndex {
+  ChangeSubIndex(
+      Schema<ChangeData> schema,
+      SitePaths sitePaths,
+      Path path,
+      GerritIndexWriterConfig writerConfig,
+      SearcherFactory searcherFactory) throws IOException {
+    this(schema, sitePaths, FSDirectory.open(path),
+        path.getFileName().toString(), writerConfig, searcherFactory);
+  }
+
+  ChangeSubIndex(
+      Schema<ChangeData> schema,
+      SitePaths sitePaths,
+      Directory dir,
+      String name,
+      GerritIndexWriterConfig writerConfig,
+      SearcherFactory searcherFactory) throws IOException {
+    super(schema, sitePaths, dir, name, writerConfig, searcherFactory);
+  }
+
+  @Override
+  public void replace(ChangeData obj) throws IOException {
+    throw new UnsupportedOperationException(
+        "don't use ChangeSubIndex directly");
+  }
+
+  @Override
+  public void delete(Change.Id key) throws IOException {
+    throw new UnsupportedOperationException(
+        "don't use ChangeSubIndex directly");
+  }
+
+  @Override
+  public DataSource<ChangeData> getSource(Predicate<ChangeData> p,
+      QueryOptions opts) throws QueryParseException {
+    throw new UnsupportedOperationException(
+        "don't use ChangeSubIndex directly");
+  }
+
+  @Override
+  void add(Document doc, Values<ChangeData> values) {
+    // Add separate DocValues fields for those fields needed for sorting.
+    FieldDef<ChangeData, ?> f = values.getField();
+    if (f == ChangeField.LEGACY_ID) {
+      int v = (Integer) getOnlyElement(values.getValues());
+      doc.add(new NumericDocValuesField(ID_SORT_FIELD, v));
+    } else if (f == ChangeField.UPDATED) {
+      long t = ((Timestamp) getOnlyElement(values.getValues())).getTime();
+      doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t));
+    }
+    super.add(doc, values);
+  }
+}

gerrit-lucene/src/main/java/com/google/gerrit/lucene/GerritIndexWriterConfig.java

@@ -0,0 +1,77 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.lucene;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.gerrit.server.config.ConfigUtil;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.eclipse.jgit.lib.Config;
+
+import java.util.Map;
+
+/**
+ * Combination of Lucene {@link IndexWriterConfig} with additional
+ * Gerrit-specific options.
+ */
+class GerritIndexWriterConfig {
+  private static final Map<String, String> CUSTOM_CHAR_MAPPING =
+      ImmutableMap.of("_", " ", ".", " ");
+
+  private final IndexWriterConfig luceneConfig;
+  private long commitWithinMs;
+  private final CustomMappingAnalyzer analyzer;
+
+  GerritIndexWriterConfig(Config cfg, String name) {
+    analyzer =
+        new CustomMappingAnalyzer(new StandardAnalyzer(
+            CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING);
+    luceneConfig = new IndexWriterConfig(analyzer)
+        .setOpenMode(OpenMode.CREATE_OR_APPEND)
+        .setCommitOnClose(true);
+    double m = 1 << 20;
+    luceneConfig.setRAMBufferSizeMB(cfg.getLong(
+        "index", name, "ramBufferSize",
+        (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB * m)) / m);
+    luceneConfig.setMaxBufferedDocs(cfg.getInt(
+        "index", name, "maxBufferedDocs",
+        IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS));
+    try {
+      commitWithinMs =
+          ConfigUtil.getTimeUnit(cfg, "index", name, "commitWithin",
+              MILLISECONDS.convert(5, MINUTES), MILLISECONDS);
+    } catch (IllegalArgumentException e) {
+      commitWithinMs = cfg.getLong("index", name, "commitWithin", 0);
+    }
+  }
+
+  CustomMappingAnalyzer getAnalyzer() {
+    return analyzer;
+  }
+
+  IndexWriterConfig getLuceneConfig() {
+    return luceneConfig;
+  }
+
+  long getCommitWithinMs() {
+    return commitWithinMs;
+  }
+}

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java

@@ -15,17 +15,15 @@
 package com.google.gerrit.lucene;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.collect.Iterables.getOnlyElement;
+import static com.google.gerrit.lucene.AbstractLuceneIndex.sortFieldName;
+import static com.google.gerrit.lucene.LuceneVersionManager.CHANGES_PREFIX;
 import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE;
 import static com.google.gerrit.server.index.change.ChangeField.CHANGE;
 import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
 import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
 import static com.google.gerrit.server.index.change.IndexRewriter.CLOSED_STATUSES;
 import static com.google.gerrit.server.index.change.IndexRewriter.OPEN_STATUSES;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.MINUTES;
 
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
@@ -37,16 +35,12 @@ import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.reviewdb.client.PatchSet;
 import com.google.gerrit.reviewdb.client.Project;
 import com.google.gerrit.reviewdb.server.ReviewDb;
-import com.google.gerrit.server.config.ConfigUtil;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.SitePaths;
-import com.google.gerrit.server.index.FieldDef;
 import com.google.gerrit.server.index.FieldDef.FillArgs;
-import com.google.gerrit.server.index.FieldType;
 import com.google.gerrit.server.index.IndexExecutor;
 import com.google.gerrit.server.index.QueryOptions;
 import com.google.gerrit.server.index.Schema;
-import com.google.gerrit.server.index.Schema.Values;
 import com.google.gerrit.server.index.change.ChangeField;
 import com.google.gerrit.server.index.change.ChangeField.ChangeProtoField;
 import com.google.gerrit.server.index.change.ChangeField.PatchSetApprovalProtoField;
@@ -64,23 +58,10 @@ import com.google.inject.Provider;
 import com.google.inject.assistedinject.Assisted;
 import com.google.inject.assistedinject.AssistedInject;
 
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.document.LongField;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
@@ -92,20 +73,16 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.BytesRef;
-import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
@@ -124,6 +101,11 @@ public class LuceneChangeIndex implements ChangeIndex {
   public static final String CHANGES_OPEN = "open";
   public static final String CHANGES_CLOSED = "closed";
 
+  static final String UPDATED_SORT_FIELD =
+      sortFieldName(ChangeField.UPDATED);
+  static final String ID_SORT_FIELD =
+      sortFieldName(ChangeField.LEGACY_ID);
+
   private static final String ADDED_FIELD = ChangeField.ADDED.getName();
   private static final String APPROVAL_FIELD = ChangeField.APPROVAL.getName();
   private static final String CHANGE_FIELD = ChangeField.CHANGE.getName();
@@ -132,68 +114,13 @@ public class LuceneChangeIndex implements ChangeIndex {
   private static final String PATCH_SET_FIELD = ChangeField.PATCH_SET.getName();
   private static final String REVIEWEDBY_FIELD =
       ChangeField.REVIEWEDBY.getName();
-  private static final String UPDATED_SORT_FIELD =
-      sortFieldName(ChangeField.UPDATED);
-  private static final String ID_SORT_FIELD =
-      sortFieldName(ChangeField.LEGACY_ID);
 
-  private static final Map<String, String> CUSTOM_CHAR_MAPPING = ImmutableMap.of(
+  static Term idTerm(ChangeData cd) {
-      "_", " ", ".", " ");
+    return QueryBuilder.intTerm(LEGACY_ID.getName(), cd.getId().get());
-
-  public static void setReady(SitePaths sitePaths, int version, boolean ready)
-      throws IOException {
-    try {
-      FileBasedConfig cfg =
-          LuceneVersionManager.loadGerritIndexConfig(sitePaths);
-      LuceneVersionManager.setReady(cfg, version, ready);
-      cfg.save();
-    } catch (ConfigInvalidException e) {
-      throw new IOException(e);
-    }
   }
 
-  private static String sortFieldName(FieldDef<?, ?> f) {
+  static Term idTerm(Change.Id id) {
-    return f.getName() + "_SORT";
+    return QueryBuilder.intTerm(LEGACY_ID.getName(), id.get());
-  }
-
-  static interface Factory {
-    LuceneChangeIndex create(Schema<ChangeData> schema);
-  }
-
-  static class GerritIndexWriterConfig {
-    private final IndexWriterConfig luceneConfig;
-    private long commitWithinMs;
-
-    private GerritIndexWriterConfig(Config cfg, String name) {
-      CustomMappingAnalyzer analyzer =
-          new CustomMappingAnalyzer(new StandardAnalyzer(
-              CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING);
-      luceneConfig = new IndexWriterConfig(analyzer)
-          .setOpenMode(OpenMode.CREATE_OR_APPEND)
-          .setCommitOnClose(true);
-      double m = 1 << 20;
-      luceneConfig.setRAMBufferSizeMB(cfg.getLong(
-          "index", name, "ramBufferSize",
-          (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB * m)) / m);
-      luceneConfig.setMaxBufferedDocs(cfg.getInt(
-          "index", name, "maxBufferedDocs",
-          IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS));
-      try {
-        commitWithinMs =
-            ConfigUtil.getTimeUnit(cfg, "index", name, "commitWithin",
-                MILLISECONDS.convert(5, MINUTES), MILLISECONDS);
-      } catch (IllegalArgumentException e) {
-        commitWithinMs = cfg.getLong("index", name, "commitWithin", 0);
-      }
-    }
-
-    IndexWriterConfig getLuceneConfig() {
-      return luceneConfig;
-    }
-
-    long getCommitWithinMs() {
-      return commitWithinMs;
-    }
   }
 
   private final SitePaths sitePaths;
@@ -202,9 +129,9 @@ public class LuceneChangeIndex implements ChangeIndex {
   private final Provider<ReviewDb> db;
   private final ChangeData.Factory changeDataFactory;
   private final Schema<ChangeData> schema;
-  private final QueryBuilder queryBuilder;
+  private final QueryBuilder<ChangeData> queryBuilder;
-  private final SubIndex openIndex;
+  private final ChangeSubIndex openIndex;
-  private final SubIndex closedIndex;
+  private final ChangeSubIndex closedIndex;
 
   @AssistedInject
   LuceneChangeIndex(
@@ -222,31 +149,25 @@ public class LuceneChangeIndex implements ChangeIndex {
     this.changeDataFactory = changeDataFactory;
     this.schema = schema;
 
-    CustomMappingAnalyzer analyzer =
-        new CustomMappingAnalyzer(new StandardAnalyzer(CharArraySet.EMPTY_SET),
-            CUSTOM_CHAR_MAPPING);
-    queryBuilder = new QueryBuilder(analyzer);
-
-    BooleanQuery.setMaxClauseCount(cfg.getInt("index", "maxTerms",
-        BooleanQuery.getMaxClauseCount()));
-
     GerritIndexWriterConfig openConfig =
         new GerritIndexWriterConfig(cfg, "changes_open");
     GerritIndexWriterConfig closedConfig =
         new GerritIndexWriterConfig(cfg, "changes_closed");
 
+    queryBuilder = new QueryBuilder<>(schema, openConfig.getAnalyzer());
+
     SearcherFactory searcherFactory = new SearcherFactory();
-    if (cfg.getBoolean("index", "lucene", "testInmemory", false)) {
+    if (LuceneIndexModule.isInMemoryTest(cfg)) {
-      openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig,
+      openIndex = new ChangeSubIndex(schema, sitePaths, new RAMDirectory(),
-          searcherFactory);
+          "ramOpen", openConfig, searcherFactory);
-      closedIndex = new SubIndex(new RAMDirectory(), "ramClosed", closedConfig,
+      closedIndex = new ChangeSubIndex(schema, sitePaths, new RAMDirectory(),
-          searcherFactory);
+          "ramClosed", closedConfig, searcherFactory);
     } else {
-      Path dir = LuceneVersionManager.getDir(sitePaths, schema);
+      Path dir = LuceneVersionManager.getDir(sitePaths, CHANGES_PREFIX, schema);
-      openIndex = new SubIndex(dir.resolve(CHANGES_OPEN), openConfig,
+      openIndex = new ChangeSubIndex(schema, sitePaths,
-          searcherFactory);
+          dir.resolve(CHANGES_OPEN), openConfig, searcherFactory);
-      closedIndex = new SubIndex(dir.resolve(CHANGES_CLOSED), closedConfig,
+      closedIndex = new ChangeSubIndex(schema, sitePaths,
-          searcherFactory);
+          dir.resolve(CHANGES_CLOSED), closedConfig, searcherFactory);
     }
   }
 
@@ -275,8 +196,10 @@ public class LuceneChangeIndex implements ChangeIndex {
 
   @Override
   public void replace(ChangeData cd) throws IOException {
-    Term id = QueryBuilder.idTerm(cd);
+    Term id = LuceneChangeIndex.idTerm(cd);
-    Document doc = toDocument(cd);
+    // toDocument is essentially static and doesn't depend on the specific
+    // sub-index, so just pick one.
+    Document doc = openIndex.toDocument(cd, fillArgs);
     try {
       if (cd.change().getStatus().isOpen()) {
         Futures.allAsList(
@@ -294,7 +217,7 @@ public class LuceneChangeIndex implements ChangeIndex {
 
   @Override
   public void delete(Change.Id id) throws IOException {
-    Term idTerm = QueryBuilder.idTerm(id);
+    Term idTerm = LuceneChangeIndex.idTerm(id);
     try {
       Futures.allAsList(
           openIndex.delete(idTerm),
@@ -314,7 +237,7 @@ public class LuceneChangeIndex implements ChangeIndex {
   public ChangeDataSource getSource(Predicate<ChangeData> p, QueryOptions opts)
       throws QueryParseException {
     Set<Change.Status> statuses = IndexRewriter.getPossibleStatus(p);
-    List<SubIndex> indexes = Lists.newArrayListWithCapacity(2);
+    List<ChangeSubIndex> indexes = Lists.newArrayListWithCapacity(2);
     if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
       indexes.add(openIndex);
     }
@@ -326,7 +249,9 @@ public class LuceneChangeIndex implements ChangeIndex {
 
   @Override
   public void markReady(boolean ready) throws IOException {
-    setReady(sitePaths, schema.getVersion(), ready);
+    // Do not delegate to ChangeSubIndex#markReady, since changes have an
+    // additional level of directory nesting.
+    AbstractLuceneIndex.setReady(sitePaths, schema.getVersion(), ready);
   }
 
   private Sort getSort() {
@@ -335,21 +260,17 @@ public class LuceneChangeIndex implements ChangeIndex {
         new SortField(ID_SORT_FIELD, SortField.Type.LONG, true));
   }
 
-  public SubIndex getOpenChangesIndex() {
+  public ChangeSubIndex getClosedChangesIndex() {
-    return openIndex;
-  }
-
-  public SubIndex getClosedChangesIndex() {
     return closedIndex;
   }
 
   private class QuerySource implements ChangeDataSource {
-    private final List<SubIndex> indexes;
+    private final List<ChangeSubIndex> indexes;
     private final Query query;
     private final QueryOptions opts;
     private final Sort sort;
 
-    private QuerySource(List<SubIndex> indexes, Query query, QueryOptions opts,
+    private QuerySource(List<ChangeSubIndex> indexes, Query query, QueryOptions opts,
         Sort sort) {
       this.indexes = indexes;
       this.query = checkNotNull(query, "null query from Lucene");
@@ -557,64 +478,4 @@ public class LuceneChangeIndex implements ChangeIndex {
     }
     return result;
   }
-
-  private Document toDocument(ChangeData cd) {
-    Document result = new Document();
-    for (Values<ChangeData> vs : schema.buildFields(cd, fillArgs)) {
-      if (vs.getValues() != null) {
-        add(result, vs);
-      }
-    }
-    return result;
-  }
-
-  private void add(Document doc, Values<ChangeData> values) {
-    String name = values.getField().getName();
-    FieldType<?> type = values.getField().getType();
-    Store store = store(values.getField());
-
-    FieldDef<ChangeData, ?> f = values.getField();
-
-    // Add separate DocValues fields for those fields needed for sorting.
-    if (f == ChangeField.LEGACY_ID) {
-      int v = (Integer) getOnlyElement(values.getValues());
-      doc.add(new NumericDocValuesField(sortFieldName(f), v));
-    } else if (f == ChangeField.UPDATED) {
-      long t = ((Timestamp) getOnlyElement(values.getValues())).getTime();
-      doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t));
-    }
-
-    if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) {
-      for (Object value : values.getValues()) {
-        doc.add(new IntField(name, (Integer) value, store));
-      }
-    } else if (type == FieldType.LONG) {
-      for (Object value : values.getValues()) {
-        doc.add(new LongField(name, (Long) value, store));
-      }
-    } else if (type == FieldType.TIMESTAMP) {
-      for (Object value : values.getValues()) {
-        doc.add(new LongField(name, ((Timestamp) value).getTime(), store));
-      }
-    } else if (type == FieldType.EXACT
-        || type == FieldType.PREFIX) {
-      for (Object value : values.getValues()) {
-        doc.add(new StringField(name, (String) value, store));
-      }
-    } else if (type == FieldType.FULL_TEXT) {
-      for (Object value : values.getValues()) {
-        doc.add(new TextField(name, (String) value, store));
-      }
-    } else if (type == FieldType.STORED_ONLY) {
-      for (Object value : values.getValues()) {
-        doc.add(new StoredField(name, (byte[]) value));
-      }
-    } else {
-      throw FieldType.badFieldType(type);
-    }
-  }
-
-  private static Field.Store store(FieldDef<?, ?> f) {
-    return f.isStored() ? Field.Store.YES : Field.Store.NO;
-  }
 }

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java

@@ -14,40 +14,69 @@
 
 package com.google.gerrit.lucene;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.gerrit.extensions.events.LifecycleListener;
 import com.google.gerrit.lifecycle.LifecycleModule;
 import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.index.Index;
 import com.google.gerrit.server.index.IndexConfig;
+import com.google.gerrit.server.index.IndexDefinition;
 import com.google.gerrit.server.index.IndexModule;
 import com.google.gerrit.server.index.Schema;
-import com.google.gerrit.server.index.change.ChangeIndexCollection;
+import com.google.gerrit.server.index.change.ChangeIndex;
-import com.google.gerrit.server.index.change.ChangeSchemas;
-import com.google.gerrit.server.query.change.ChangeData;
 import com.google.inject.Inject;
 import com.google.inject.Provides;
+import com.google.inject.ProvisionException;
 import com.google.inject.Singleton;
+import com.google.inject.TypeLiteral;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.google.inject.name.Named;
+import com.google.inject.name.Names;
 
+import org.apache.lucene.search.BooleanQuery;
 import org.eclipse.jgit.lib.Config;
 
-public class LuceneIndexModule extends LifecycleModule {
+import java.util.Collection;
-  private final Integer singleVersion;
+import java.util.Map;
-  private final int threads;
 
-  public LuceneIndexModule() {
+public class LuceneIndexModule extends LifecycleModule {
-    this(null, 0);
+  private static final String SINGLE_VERSIONS =
+      "LuceneIndexModule/SingleVersions";
+
+  public static LuceneIndexModule singleVersionAllLatest(int threads) {
+    return new LuceneIndexModule(ImmutableMap.<String, Integer> of(), threads);
   }
 
-  public LuceneIndexModule(Integer singleVersion, int threads) {
+  public static LuceneIndexModule singleVersionWithExplicitVersions(
-    this.singleVersion = singleVersion;
+      Map<String, Integer> versions, int threads) {
+    return new LuceneIndexModule(versions, threads);
+  }
+
+  public static LuceneIndexModule latestVersionWithOnlineUpgrade() {
+    return new LuceneIndexModule(null, 0);
+  }
+
+  static boolean isInMemoryTest(Config cfg) {
+    return cfg.getBoolean("index", "lucene", "testInmemory", false);
+  }
+
+  private final int threads;
+  private final Map<String, Integer> singleVersions;
+
+  private LuceneIndexModule(Map<String, Integer> singleVersions, int threads) {
+    this.singleVersions = singleVersions;
     this.threads = threads;
   }
 
   @Override
   protected void configure() {
-    factory(LuceneChangeIndex.Factory.class);
+    install(
-    factory(OnlineReindexer.Factory.class);
+        new FactoryModuleBuilder()
+            .implement(ChangeIndex.class, LuceneChangeIndex.class)
+            .build(ChangeIndex.Factory.class));
+
     install(new IndexModule(threads));
-    if (singleVersion == null) {
+    if (singleVersions == null) {
       install(new MultiVersionModule());
     } else {
       install(new SingleVersionModule());
@@ -57,6 +86,8 @@ public class LuceneIndexModule extends LifecycleModule {
   @Provides
   @Singleton
   IndexConfig getIndexConfig(@GerritServerConfig Config cfg) {
+    BooleanQuery.setMaxClauseCount(cfg.getInt("index", "maxTerms",
+        BooleanQuery.getMaxClauseCount()));
     return IndexConfig.fromConfig(cfg);
   }
 
@@ -71,34 +102,48 @@ public class LuceneIndexModule extends LifecycleModule {
     @Override
     public void configure() {
       listener().to(SingleVersionListener.class);
-    }
+      bind(new TypeLiteral<Map<String, Integer>>() {})
-
+          .annotatedWith(Names.named(SINGLE_VERSIONS))
-    @Provides
+          .toInstance(singleVersions);
-    @Singleton
-    LuceneChangeIndex getIndex(LuceneChangeIndex.Factory factory) {
-      Schema<ChangeData> schema = singleVersion != null
-          ? ChangeSchemas.get(singleVersion)
-          : ChangeSchemas.getLatest();
-      return factory.create(schema);
     }
   }
 
   @Singleton
   static class SingleVersionListener implements LifecycleListener {
-    private final ChangeIndexCollection indexes;
+    private final Collection<IndexDefinition<?, ?, ?>> defs;
-    private final LuceneChangeIndex index;
+    private final Map<String, Integer> singleVersions;
 
     @Inject
-    SingleVersionListener(ChangeIndexCollection indexes,
+    SingleVersionListener(
-        LuceneChangeIndex index) {
+        Collection<IndexDefinition<?, ?, ?>> defs,
-      this.indexes = indexes;
+        @Named(SINGLE_VERSIONS) Map<String, Integer> singleVersions) {
-      this.index = index;
+      this.defs = defs;
+      this.singleVersions = singleVersions;
     }
 
     @Override
     public void start() {
-      indexes.setSearchIndex(index);
+      for (IndexDefinition<?, ?, ?> def : defs) {
-      indexes.addWriteIndex(index);
+        start(def);
+      }
+    }
+
+    private <K, V, I extends Index<K, V>> void start(
+        IndexDefinition<K, V, I> def) {
+      Schema<V> schema;
+      Integer v = singleVersions.get(def.getName());
+      if (v == null) {
+        schema = def.getLatest();
+      } else {
+        schema = def.getSchemas().get(v);
+        if (schema == null) {
+          throw new ProvisionException(String.format(
+                "Unrecognized %s schema version: %s", def.getName(), v));
+        }
+      }
+      I index = def.getIndexFactory().create(schema);
+      def.getIndexCollection().setSearchIndex(index);
+      def.getIndexCollection().addWriteIndex(index);
     }
 
     @Override

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java

@@ -20,11 +20,14 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.primitives.Ints;
 import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.OnlineReindexer;
 import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.index.change.ChangeIndex;
 import com.google.gerrit.server.index.change.ChangeIndexCollection;
-import com.google.gerrit.server.index.change.ChangeSchemas;
+import com.google.gerrit.server.index.change.ChangeIndexDefintion;
 import com.google.gerrit.server.query.change.ChangeData;
 import com.google.inject.Inject;
 import com.google.inject.ProvisionException;
@@ -50,7 +53,7 @@ public class LuceneVersionManager implements LifecycleListener {
   private static final Logger log = LoggerFactory
       .getLogger(LuceneVersionManager.class);
 
-  private static final String CHANGES_PREFIX = "changes_";
+  static final String CHANGES_PREFIX = "changes_";
 
   private static class Version {
     private final Schema<ChangeData> schema;
@@ -68,9 +71,9 @@ public class LuceneVersionManager implements LifecycleListener {
     }
   }
 
-  static Path getDir(SitePaths sitePaths, Schema<ChangeData> schema) {
+  static Path getDir(SitePaths sitePaths, String prefix, Schema<?> schema) {
     return sitePaths.index_dir.resolve(String.format("%s%04d",
-        CHANGES_PREFIX, schema.getVersion()));
+        prefix, schema.getVersion()));
   }
 
   static FileBasedConfig loadGerritIndexConfig(SitePaths sitePaths)
@@ -93,9 +96,9 @@ public class LuceneVersionManager implements LifecycleListener {
   private final SitePaths sitePaths;
   private final LuceneChangeIndex.Factory indexFactory;
   private final ChangeIndexCollection indexes;
-  private final OnlineReindexer.Factory reindexerFactory;
+  private final ChangeIndexDefintion changeDef;
   private final boolean onlineUpgrade;
-  private OnlineReindexer reindexer;
+  private OnlineReindexer<Change.Id, ChangeData, ChangeIndex> reindexer;
 
   @Inject
   LuceneVersionManager(
@@ -103,11 +106,11 @@ public class LuceneVersionManager implements LifecycleListener {
       SitePaths sitePaths,
       LuceneChangeIndex.Factory indexFactory,
       ChangeIndexCollection indexes,
-      OnlineReindexer.Factory reindexerFactory) {
+      ChangeIndexDefintion changeDef) {
     this.sitePaths = sitePaths;
     this.indexFactory = indexFactory;
     this.indexes = indexes;
-    this.reindexerFactory = reindexerFactory;
+    this.changeDef = changeDef;
     this.onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true);
   }
 
@@ -157,7 +160,8 @@ public class LuceneVersionManager implements LifecycleListener {
     }
 
     markNotReady(cfg, versions.values(), write);
-    LuceneChangeIndex searchIndex = indexFactory.create(search.schema);
+    LuceneChangeIndex searchIndex =
+        (LuceneChangeIndex) indexFactory.create(search.schema);
     indexes.setSearchIndex(searchIndex);
     for (Version v : write) {
       if (v.schema != null) {
@@ -171,7 +175,7 @@ public class LuceneVersionManager implements LifecycleListener {
 
     int latest = write.get(0).version;
     if (onlineUpgrade && latest != search.version) {
-      reindexer = reindexerFactory.create(latest);
+      reindexer = new OnlineReindexer<>(changeDef, latest);
       reindexer.start();
     }
   }
@@ -223,8 +227,8 @@ public class LuceneVersionManager implements LifecycleListener {
 
   private TreeMap<Integer, Version> scanVersions(Config cfg) {
     TreeMap<Integer, Version> versions = Maps.newTreeMap();
-    for (Schema<ChangeData> schema : ChangeSchemas.ALL.values()) {
+    for (Schema<ChangeData> schema : changeDef.getSchemas().values()) {
-      Path p = getDir(sitePaths, schema);
+      Path p = getDir(sitePaths, CHANGES_PREFIX, schema);
       boolean isDir = Files.isDirectory(p);
       if (Files.exists(p) && !isDir) {
         log.warn("Not a directory: %s", p.toAbsolutePath());

gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java

@@ -14,24 +14,23 @@
 
 package com.google.gerrit.lucene;
 
-import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
+import static com.google.common.base.Preconditions.checkArgument;
 import static org.apache.lucene.search.BooleanClause.Occur.MUST;
 import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
 import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
 
 import com.google.common.collect.Lists;
-import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.server.index.FieldType;
 import com.google.gerrit.server.index.IndexPredicate;
 import com.google.gerrit.server.index.IntegerRangePredicate;
 import com.google.gerrit.server.index.RegexPredicate;
+import com.google.gerrit.server.index.Schema;
 import com.google.gerrit.server.index.TimestampRangePredicate;
 import com.google.gerrit.server.query.AndPredicate;
 import com.google.gerrit.server.query.NotPredicate;
 import com.google.gerrit.server.query.OrPredicate;
 import com.google.gerrit.server.query.Predicate;
 import com.google.gerrit.server.query.QueryParseException;
-import com.google.gerrit.server.query.change.ChangeData;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.Term;
@@ -48,23 +47,22 @@ import org.apache.lucene.util.NumericUtils;
 import java.util.Date;
 import java.util.List;
 
-public class QueryBuilder {
+public class QueryBuilder<V> {
-
+  static Term intTerm(String name, int value) {
-  public static Term idTerm(ChangeData cd) {
+    BytesRefBuilder builder = new BytesRefBuilder();
-    return intTerm(LEGACY_ID.getName(), cd.getId().get());
+    NumericUtils.intToPrefixCodedBytes(value, 0, builder);
-  }
+    return new Term(name, builder.get());
-
-  public static Term idTerm(Change.Id id) {
-    return intTerm(LEGACY_ID.getName(), id.get());
   }
 
+  private final Schema<V> schema;
   private final org.apache.lucene.util.QueryBuilder queryBuilder;
 
-  public QueryBuilder(Analyzer analyzer) {
+  public QueryBuilder(Schema<V> schema, Analyzer analyzer) {
+    this.schema = schema;
     queryBuilder = new org.apache.lucene.util.QueryBuilder(analyzer);
   }
 
-  public Query toQuery(Predicate<ChangeData> p) throws QueryParseException {
+  public Query toQuery(Predicate<V> p) throws QueryParseException {
     if (p instanceof AndPredicate) {
       return and(p);
     } else if (p instanceof OrPredicate) {
@@ -72,13 +70,13 @@ public class QueryBuilder {
     } else if (p instanceof NotPredicate) {
       return not(p);
     } else if (p instanceof IndexPredicate) {
-      return fieldQuery((IndexPredicate<ChangeData>) p);
+      return fieldQuery((IndexPredicate<V>) p);
     } else {
       throw new QueryParseException("cannot create query for index: " + p);
     }
   }
 
-  private Query or(Predicate<ChangeData> p)
+  private Query or(Predicate<V> p)
       throws QueryParseException {
     try {
       BooleanQuery.Builder q = new BooleanQuery.Builder();
@@ -91,17 +89,17 @@ public class QueryBuilder {
     }
   }
 
-  private Query and(Predicate<ChangeData> p)
+  private Query and(Predicate<V> p)
       throws QueryParseException {
     try {
       BooleanQuery.Builder b = new BooleanQuery.Builder();
       List<Query> not = Lists.newArrayListWithCapacity(p.getChildCount());
       for (int i = 0; i < p.getChildCount(); i++) {
-        Predicate<ChangeData> c = p.getChild(i);
+        Predicate<V> c = p.getChild(i);
         if (c instanceof NotPredicate) {
-          Predicate<ChangeData> n = c.getChild(0);
+          Predicate<V> n = c.getChild(0);
           if (n instanceof TimestampRangePredicate) {
-            b.add(notTimestamp((TimestampRangePredicate<ChangeData>) n), MUST);
+            b.add(notTimestamp((TimestampRangePredicate<V>) n), MUST);
           } else {
             not.add(toQuery(n));
           }
@@ -118,11 +116,11 @@ public class QueryBuilder {
     }
   }
 
-  private Query not(Predicate<ChangeData> p)
+  private Query not(Predicate<V> p)
       throws QueryParseException {
-    Predicate<ChangeData> n = p.getChild(0);
+    Predicate<V> n = p.getChild(0);
     if (n instanceof TimestampRangePredicate) {
-      return notTimestamp((TimestampRangePredicate<ChangeData>) n);
+      return notTimestamp((TimestampRangePredicate<V>) n);
     }
 
     // Lucene does not support negation, start with all and subtract.
@@ -132,8 +130,11 @@ public class QueryBuilder {
       .build();
   }
 
-  private Query fieldQuery(IndexPredicate<ChangeData> p)
+  private Query fieldQuery(IndexPredicate<V> p)
       throws QueryParseException {
+    checkArgument(schema.hasField(p.getField()),
+        "field not in schema v%s: %s", schema.getVersion(),
+        p.getField().getName());
     if (p.getType() == FieldType.INTEGER) {
       return intQuery(p);
     } else if (p.getType() == FieldType.INTEGER_RANGE) {
@@ -151,13 +152,7 @@ public class QueryBuilder {
     }
   }
 
-  private static Term intTerm(String name, int value) {
+  private Query intQuery(IndexPredicate<V> p)
-    BytesRefBuilder builder = new BytesRefBuilder();
-    NumericUtils.intToPrefixCodedBytes(value, 0, builder);
-    return new Term(name, builder.get());
-  }
-
-  private Query intQuery(IndexPredicate<ChangeData> p)
       throws QueryParseException {
     int value;
     try {
@@ -170,11 +165,11 @@ public class QueryBuilder {
     return new TermQuery(intTerm(p.getField().getName(), value));
   }
 
-  private Query intRangeQuery(IndexPredicate<ChangeData> p)
+  private Query intRangeQuery(IndexPredicate<V> p)
       throws QueryParseException {
     if (p instanceof IntegerRangePredicate) {
-      IntegerRangePredicate<ChangeData> r =
+      IntegerRangePredicate<V> r =
-          (IntegerRangePredicate<ChangeData>) p;
+          (IntegerRangePredicate<V>) p;
       int minimum = r.getMinimumValue();
       int maximum = r.getMaximumValue();
       if (minimum == maximum) {
@@ -192,11 +187,11 @@ public class QueryBuilder {
     throw new QueryParseException("not an integer range: " + p);
   }
 
-  private Query timestampQuery(IndexPredicate<ChangeData> p)
+  private Query timestampQuery(IndexPredicate<V> p)
       throws QueryParseException {
     if (p instanceof TimestampRangePredicate) {
-      TimestampRangePredicate<ChangeData> r =
+      TimestampRangePredicate<V> r =
-          (TimestampRangePredicate<ChangeData>) p;
+          (TimestampRangePredicate<V>) p;
       return NumericRangeQuery.newLongRange(
           r.getField().getName(),
           r.getMinTimestamp().getTime(),
@@ -206,7 +201,7 @@ public class QueryBuilder {
     throw new QueryParseException("not a timestamp: " + p);
   }
 
-  private Query notTimestamp(TimestampRangePredicate<ChangeData> r)
+  private Query notTimestamp(TimestampRangePredicate<V> r)
       throws QueryParseException {
     if (r.getMinTimestamp().getTime() == 0) {
       return NumericRangeQuery.newLongRange(
@@ -218,7 +213,7 @@ public class QueryBuilder {
     throw new QueryParseException("cannot negate: " + r);
   }
 
-  private Query exactQuery(IndexPredicate<ChangeData> p) {
+  private Query exactQuery(IndexPredicate<V> p) {
     if (p instanceof RegexPredicate<?>) {
       return regexQuery(p);
     } else {
@@ -226,7 +221,7 @@ public class QueryBuilder {
     }
   }
 
-  private Query regexQuery(IndexPredicate<ChangeData> p) {
+  private Query regexQuery(IndexPredicate<V> p) {
     String re = p.getValue();
     if (re.startsWith("^")) {
       re = re.substring(1);
@@ -237,11 +232,11 @@ public class QueryBuilder {
     return new RegexpQuery(new Term(p.getField().getName(), re));
   }
 
-  private Query prefixQuery(IndexPredicate<ChangeData> p) {
+  private Query prefixQuery(IndexPredicate<V> p) {
     return new PrefixQuery(new Term(p.getField().getName(), p.getValue()));
   }
 
-  private Query fullTextQuery(IndexPredicate<ChangeData> p)
+  private Query fullTextQuery(IndexPredicate<V> p)
       throws QueryParseException {
     String value = p.getValue();
     if (value == null) {

gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java

@@ -403,7 +403,9 @@ public class Daemon extends SiteProgram {
     }
     switch (indexType) {
       case LUCENE:
-        return luceneModule != null ? luceneModule : new LuceneIndexModule();
+        return luceneModule != null
+            ? luceneModule
+            : LuceneIndexModule.latestVersionWithOnlineUpgrade();
       default:
         throw new IllegalStateException("unsupported index.type = " + indexType);
     }

gerrit-pgm/src/main/java/com/google/gerrit/pgm/Reindex.java

@@ -31,9 +31,10 @@ import com.google.gerrit.server.git.ScanningChangeCacheImpl;
 import com.google.gerrit.server.index.IndexModule;
 import com.google.gerrit.server.index.IndexModule.IndexType;
 import com.google.gerrit.server.index.SiteIndexer;
+import com.google.gerrit.server.index.change.AllChangesIndexer;
 import com.google.gerrit.server.index.change.ChangeIndex;
 import com.google.gerrit.server.index.change.ChangeIndexCollection;
-import com.google.gerrit.server.index.change.ChangeSchemas;
+import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
 import com.google.gerrit.server.notedb.ChangeNotes;
 import com.google.gerrit.server.project.ProjectCache;
 import com.google.inject.Injector;
@@ -47,7 +48,9 @@ import org.eclipse.jgit.lib.TextProgressMonitor;
 import org.eclipse.jgit.util.io.NullOutputStream;
 import org.kohsuke.args4j.Option;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -55,9 +58,9 @@ public class Reindex extends SiteProgram {
   @Option(name = "--threads", usage = "Number of threads to use for indexing")
   private int threads = Runtime.getRuntime().availableProcessors();
 
-  @Option(name = "--schema-version",
+  @Option(name = "--changes-schema-version",
-      usage = "Schema version to reindex; default is most recent version")
+      usage = "Schema version to reindex, for changes; default is most recent version")
-  private Integer version;
+  private Integer changesVersion;
 
   @Option(name = "--verbose", usage = "Output debug information for each change")
   private boolean verbose;
@@ -82,9 +85,6 @@ public class Reindex extends SiteProgram {
     checkNotSlaveMode();
     disableLuceneAutomaticCommit();
     disableChangeCache();
-    if (version == null) {
-      version = ChangeSchemas.getLatest().getVersion();
-    }
     LifecycleManager dbManager = new LifecycleManager();
     dbManager.add(dbInjector);
     dbManager.start();
@@ -120,11 +120,16 @@ public class Reindex extends SiteProgram {
   }
 
   private Injector createSysInjector() {
+    Map<String, Integer> versions = new HashMap<>();
+    if (changesVersion != null) {
+      versions.put(ChangeSchemaDefinitions.INSTANCE.getName(), changesVersion);
+    }
     List<Module> modules = Lists.newArrayList();
     Module changeIndexModule;
     switch (IndexModule.getIndexType(dbInjector)) {
       case LUCENE:
-        changeIndexModule = new LuceneIndexModule(version, threads);
+        changeIndexModule = LuceneIndexModule.singleVersionWithExplicitVersions(
+            versions, threads);
         break;
       default:
         throw new IllegalStateException("unsupported index.type");
@@ -164,9 +169,9 @@ public class Reindex extends SiteProgram {
     }
     pm.endTask();
 
-    SiteIndexer batchIndexer =
+    AllChangesIndexer batchIndexer =
-        sysInjector.getInstance(SiteIndexer.class);
+        sysInjector.getInstance(AllChangesIndexer.class);
-    SiteIndexer.Result result = batchIndexer.setNumChanges(changeCount)
+    SiteIndexer.Result result = batchIndexer.setTotalWork(changeCount)
         .setProgressOut(System.err)
         .setVerboseOut(verbose ? System.out : NullOutputStream.INSTANCE)
         .indexAll(index, projects);

gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitIndex.java

@@ -15,14 +15,15 @@
 package com.google.gerrit.pgm.init;
 
 import com.google.common.collect.Iterables;
-import com.google.gerrit.lucene.LuceneChangeIndex;
+import com.google.gerrit.lucene.AbstractLuceneIndex;
 import com.google.gerrit.pgm.init.api.ConsoleUI;
 import com.google.gerrit.pgm.init.api.InitFlags;
 import com.google.gerrit.pgm.init.api.InitStep;
 import com.google.gerrit.pgm.init.api.Section;
 import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.IndexModule;
 import com.google.gerrit.server.index.IndexModule.IndexType;
-import com.google.gerrit.server.index.change.ChangeSchemas;
+import com.google.gerrit.server.index.SchemaDefinitions;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 
@@ -57,8 +58,11 @@ class InitIndex implements InitStep {
     ui.header("Index");
 
     IndexType type = index.select("Type", "type", IndexType.LUCENE);
-    LuceneChangeIndex.setReady(
+    for (SchemaDefinitions<?> def : IndexModule.ALL_SCHEMA_DEFS) {
-        site, ChangeSchemas.getLatest().getVersion(), true);
+      // TODO(dborowitz): Totally broken for non-change indexes.
+      AbstractLuceneIndex.setReady(
+          site, def.getLatest().getVersion(), true);
+    }
     if ((site.isNew || isEmptySite()) && type == IndexType.LUCENE) {
     } else {
       final String message = String.format(

gerrit-server/src/main/java/com/google/gerrit/server/index/DummyIndexModule.java

@@ -14,15 +14,24 @@
 
 package com.google.gerrit.server.index;
 
+import com.google.gerrit.server.index.change.ChangeIndex;
 import com.google.gerrit.server.index.change.DummyChangeIndex;
+import com.google.gerrit.server.query.change.ChangeData;
 import com.google.inject.AbstractModule;
 
 public class DummyIndexModule extends AbstractModule {
+  private static class DummyChangeIndexFactory implements ChangeIndex.Factory {
+    @Override
+    public ChangeIndex create(Schema<ChangeData> schema) {
+      throw new UnsupportedOperationException();
+    }
+  }
 
   @Override
   protected void configure() {
     install(new IndexModule(1));
     bind(IndexConfig.class).toInstance(IndexConfig.createDefault());
     bind(Index.class).toInstance(new DummyChangeIndex());
+    bind(ChangeIndex.Factory.class).toInstance(new DummyChangeIndexFactory());
   }
 }

gerrit-server/src/main/java/com/google/gerrit/server/index/IndexDefinition.java

@@ -0,0 +1,71 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index;
+
+import com.google.common.collect.ImmutableSortedMap;
+
+/**
+ * Definition of an index over a Gerrit data type.
+ * <p>
+ * An <em>index</em> includes a set of schema definitions along with the
+ * specific implementations used to query the secondary index implementation in
+ * a running server. If you are just interested in the static definition of one
+ * or more schemas, see the implementations of {@link SchemaDefinitions}.
+ */
+public abstract class IndexDefinition<K, V, I extends Index<K, V>> {
+  public interface IndexFactory<K, V, I extends Index<K, V>> {
+    I create(Schema<V> schema);
+  }
+
+  private final SchemaDefinitions<V> schemaDefs;
+  private final IndexCollection<K, V, I> indexCollection;
+  private final IndexFactory<K, V, I> indexFactory;
+  private final SiteIndexer<K, V, I> siteIndexer;
+
+  protected IndexDefinition(
+      SchemaDefinitions<V> schemaDefs,
+      IndexCollection<K, V, I> indexCollection,
+      IndexFactory<K, V, I> indexFactory,
+      SiteIndexer<K, V, I> siteIndexer) {
+    this.schemaDefs = schemaDefs;
+    this.indexCollection = indexCollection;
+    this.indexFactory = indexFactory;
+    this.siteIndexer = siteIndexer;
+  }
+
+  public final String getName() {
+    return schemaDefs.getName();
+  }
+
+  public final ImmutableSortedMap<Integer, Schema<V>> getSchemas() {
+    return schemaDefs.getSchemas();
+  }
+
+  public final Schema<V> getLatest() {
+    return schemaDefs.getLatest();
+  }
+
+  public final IndexCollection<K, V, I> getIndexCollection() {
+    return indexCollection;
+  }
+
+  public final IndexFactory<K, V, I> getIndexFactory() {
+    return indexFactory;
+  }
+
+  public final SiteIndexer<K, V, I> getSiteIndexer() {
+    return siteIndexer;
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/index/IndexModule.java

@@ -17,21 +17,31 @@ package com.google.gerrit.server.index;
 import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
 import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE;
 
+import com.google.common.base.Function;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.gerrit.lifecycle.LifecycleModule;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.git.WorkQueue;
 import com.google.gerrit.server.index.change.ChangeIndexCollection;
+import com.google.gerrit.server.index.change.ChangeIndexDefintion;
 import com.google.gerrit.server.index.change.ChangeIndexer;
+import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
 import com.google.gerrit.server.index.change.IndexRewriter;
 import com.google.inject.Injector;
 import com.google.inject.Key;
 import com.google.inject.Provides;
+import com.google.inject.ProvisionException;
 import com.google.inject.Singleton;
 
 import org.eclipse.jgit.lib.Config;
 
+import java.util.Collection;
+import java.util.Set;
+
 /**
  * Module for non-indexer-specific secondary index setup.
  * <p>
@@ -43,6 +53,10 @@ public class IndexModule extends LifecycleModule {
     LUCENE
   }
 
+  public static final ImmutableCollection<SchemaDefinitions<?>> ALL_SCHEMA_DEFS =
+      ImmutableList.<SchemaDefinitions<?>> of(
+          ChangeSchemaDefinitions.INSTANCE);
+
   /** Type of secondary index. */
   public static IndexType getIndexType(Injector injector) {
     Config cfg = injector.getInstance(
@@ -74,6 +88,33 @@ public class IndexModule extends LifecycleModule {
     factory(ChangeIndexer.Factory.class);
   }
 
+  @Provides
+  Collection<IndexDefinition<?, ?, ?>> getIndexDefinitions(
+      ChangeIndexDefintion changes) {
+    Collection<IndexDefinition<?, ?, ?>> result =
+        ImmutableList.<IndexDefinition<?, ?, ?>> of(changes);
+    Set<String> expected = FluentIterable.from(ALL_SCHEMA_DEFS)
+        .transform(new Function<SchemaDefinitions<?>, String>() {
+          @Override
+          public String apply(SchemaDefinitions<?> in) {
+            return in.getName();
+          }
+        }).toSet();
+    Set<String> actual = FluentIterable.from(result)
+        .transform(new Function<IndexDefinition<?, ?, ?>, String>() {
+          @Override
+          public String apply(IndexDefinition<?, ?, ?> in) {
+            return in.getName();
+          }
+        }).toSet();
+    if (!expected.equals(actual)) {
+      throw new ProvisionException(
+          "need index definitions for all schemas: "
+          + expected + " != " + actual);
+    }
+    return result;
+  }
+
   @Provides
   @Singleton
   ChangeIndexer getChangeIndexer(

gerrit-server/src/main/java/com/google/gerrit/server/index/OnlineReindexer.java

@@ -12,18 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.gerrit.lucene;
+package com.google.gerrit.server.index;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.common.collect.Lists;
-import com.google.gerrit.server.index.Index;
-import com.google.gerrit.server.index.SiteIndexer;
-import com.google.gerrit.server.index.change.ChangeIndex;
-import com.google.gerrit.server.index.change.ChangeIndexCollection;
-import com.google.gerrit.server.project.ProjectCache;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,30 +25,21 @@ import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-public class OnlineReindexer {
+public class OnlineReindexer<K, V, I extends Index<K, V>> {
   private static final Logger log = LoggerFactory
       .getLogger(OnlineReindexer.class);
 
-  public interface Factory {
+  private final IndexCollection<K, V, I> indexes;
-    OnlineReindexer create(int version);
+  private final SiteIndexer<K, V, I> batchIndexer;
-  }
-
-  private final ChangeIndexCollection indexes;
-  private final SiteIndexer batchIndexer;
-  private final ProjectCache projectCache;
   private final int version;
-  private ChangeIndex index;
+  private I index;
   private final AtomicBoolean running = new AtomicBoolean();
 
-  @Inject
+  public OnlineReindexer(
-  OnlineReindexer(
+      IndexDefinition<K, V, I> def,
-      ChangeIndexCollection indexes,
+      int version) {
-      SiteIndexer batchIndexer,
+    this.indexes = def.getIndexCollection();
-      ProjectCache projectCache,
+    this.batchIndexer = def.getSiteIndexer();
-      @Assisted int version) {
-    this.indexes = indexes;
-    this.batchIndexer = batchIndexer;
-    this.projectCache = projectCache;
     this.version = version;
   }
 
@@ -94,8 +78,7 @@ public class OnlineReindexer {
         "not an active write schema version: %s", version);
     log.info("Starting online reindex from schema version {} to {}",
         version(indexes.getSearchIndex()), version(index));
-    SiteIndexer.Result result =
+    SiteIndexer.Result result = batchIndexer.indexAll(index);
-        batchIndexer.indexAll(index, projectCache.all());
     if (!result.success()) {
       log.error("Online reindex of schema version {} failed. Successfully"
           + " indexed {} changes, failed to index {} changes",
@@ -106,7 +89,7 @@ public class OnlineReindexer {
     activateIndex();
   }
 
-  void activateIndex() {
+  public void activateIndex() {
     indexes.setSearchIndex(index);
     log.info("Using schema version {}", version(index));
     try {
@@ -115,13 +98,13 @@ public class OnlineReindexer {
       log.warn("Error activating new schema version {}", version(index));
     }
 
-    List<ChangeIndex> toRemove = Lists.newArrayListWithExpectedSize(1);
+    List<I> toRemove = Lists.newArrayListWithExpectedSize(1);
-    for (ChangeIndex i : indexes.getWriteIndexes()) {
+    for (I i : indexes.getWriteIndexes()) {
       if (version(i) != version(index)) {
         toRemove.add(i);
       }
     }
-    for (ChangeIndex i : toRemove) {
+    for (I i : toRemove) {
       try {
         i.markReady(false);
         indexes.removeWriteIndex(version(i));

gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaDefinitions.java

@@ -0,0 +1,58 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ImmutableSortedMap;
+
+/**
+ * Definitions of the various schema versions over a given Gerrit data type.
+ * <p>
+ * A <em>schema</em> is a description of the fields that are indexed over the
+ * given data type. This class contains all the versions of a schema defined
+ * over its data type, exposed as a map of version number to schema definition.
+ * If you are interested in the classes responsible for backend-specific runtime
+ * implementations, see the implementations of {@link IndexDefinition}.
+ */
+public abstract class SchemaDefinitions<V> {
+  private final String name;
+  private final ImmutableSortedMap<Integer, Schema<V>> schemas;
+
+  protected SchemaDefinitions(String name, Class<V> valueClass) {
+    this.name = checkNotNull(name);
+    this.schemas = SchemaUtil.schemasFromClass(getClass(), valueClass);
+  }
+
+  public final String getName() {
+    return name;
+  }
+
+  public final ImmutableSortedMap<Integer, Schema<V>> getSchemas() {
+    return schemas;
+  }
+
+  public final Schema<V> get(int version) {
+    Schema<V> schema = schemas.get(version);
+    checkArgument(schema != null,
+        "Unrecognized %s schema version: %s", name, version);
+    return schema;
+  }
+
+  public final Schema<V> getLatest() {
+    return schemas.lastEntry().getValue();
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/index/SchemaUtil.java

@@ -19,8 +19,8 @@ import static com.google.common.base.Preconditions.checkArgument;
 import com.google.common.base.CharMatcher;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
@@ -30,16 +30,16 @@ import org.eclipse.jgit.lib.PersonIdent;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.lang.reflect.ParameterizedType;
-import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
 public class SchemaUtil {
-  public static <V> ImmutableMap<Integer, Schema<V>> schemasFromClass(
+  public static <V> ImmutableSortedMap<Integer, Schema<V>> schemasFromClass(
       Class<?> schemasClass, Class<V> valueClass) {
-    Map<Integer, Schema<V>> schemas = Maps.newTreeMap();
+    Map<Integer, Schema<V>> schemas = Maps.newHashMap();
     for (Field f : schemasClass.getDeclaredFields()) {
       if (Modifier.isStatic(f.getModifiers())
           && Modifier.isFinal(f.getModifiers())
@@ -65,7 +65,7 @@ public class SchemaUtil {
     if (schemas.isEmpty()) {
       throw new ExceptionInInitializerError("no ChangeSchemas found");
     }
-    return ImmutableMap.copyOf(schemas);
+    return ImmutableSortedMap.copyOf(schemas);
   }
 
   public static <V> Schema<V> schema(Collection<FieldDef<V, ?>> fields) {
@@ -81,13 +81,28 @@ public class SchemaUtil {
     if (person == null) {
       return ImmutableSet.of();
     }
-    HashSet<String> parts = Sets.newHashSet();
+    return getPersonParts(
-    String email = person.getEmailAddress().toLowerCase();
+        person.getName(),
-    parts.add(email);
+        Collections.singleton(person.getEmailAddress()));
-    parts.addAll(Arrays.asList(email.split("@")));
+  }
+
+  public static Set<String> getPersonParts(String name,
+      Iterable<String> emails) {
+    Splitter at = Splitter.on('@');
     Splitter s = Splitter.on(CharMatcher.anyOf("@.- ")).omitEmptyStrings();
-    Iterables.addAll(parts, s.split(email));
+    HashSet<String> parts = Sets.newHashSet();
-    Iterables.addAll(parts, s.split(person.getName().toLowerCase()));
+    for (String email : emails) {
+      if (email == null) {
+        continue;
+      }
+      String lowerEmail = email.toLowerCase();
+      parts.add(lowerEmail);
+      Iterables.addAll(parts, at.split(lowerEmail));
+      Iterables.addAll(parts, s.split(lowerEmail));
+    }
+    if (name != null) {
+      Iterables.addAll(parts, s.split(name.toLowerCase()));
+    }
     return parts;
   }
 

gerrit-server/src/main/java/com/google/gerrit/server/index/SiteIndexer.java

@@ -1,4 +1,4 @@
-// Copyright (C) 2013 The Android Open Source Project
+// Copyright (C) 2016 The Android Open Source Project
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,81 +14,18 @@
 
 package com.google.gerrit.server.index;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
-import static org.eclipse.jgit.lib.RefDatabase.ALL;
-
 import com.google.common.base.Stopwatch;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.AsyncFunction;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.reviewdb.server.ReviewDb;
-import com.google.gerrit.server.config.GerritServerConfig;
-import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.git.MergeUtil;
-import com.google.gerrit.server.git.MultiProgressMonitor;
-import com.google.gerrit.server.git.MultiProgressMonitor.Task;
-import com.google.gerrit.server.index.change.ChangeIndex;
-import com.google.gerrit.server.index.change.ChangeIndexer;
-import com.google.gerrit.server.notedb.ChangeNotes;
-import com.google.gerrit.server.patch.PatchListLoader;
-import com.google.gerrit.server.query.change.ChangeData;
-import com.google.gwtorm.server.SchemaFactory;
-import com.google.inject.Inject;
 
-import org.eclipse.jgit.diff.DiffEntry;
-import org.eclipse.jgit.diff.DiffFormatter;
-import org.eclipse.jgit.errors.RepositoryNotFoundException;
-import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.lib.Constants;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.ObjectInserter;
-import org.eclipse.jgit.lib.ProgressMonitor;
-import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.merge.ThreeWayMergeStrategy;
-import org.eclipse.jgit.revwalk.RevCommit;
-import org.eclipse.jgit.revwalk.RevObject;
-import org.eclipse.jgit.revwalk.RevTree;
-import org.eclipse.jgit.revwalk.RevWalk;
-import org.eclipse.jgit.util.io.DisabledOutputStream;
-import org.eclipse.jgit.util.io.NullOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 
-public class SiteIndexer {
+public interface SiteIndexer<K, V, I extends Index<K, V>> {
-  private static final Logger log =
+  public class Result {
-      LoggerFactory.getLogger(SiteIndexer.class);
-
-  public static class Result {
     private final long elapsedNanos;
     private final boolean success;
     private final int done;
     private final int failed;
 
-    private Result(Stopwatch sw, boolean success, int done, int failed) {
+    public Result(Stopwatch sw, boolean success, int done, int failed) {
       this.elapsedNanos = sw.elapsed(TimeUnit.NANOSECONDS);
       this.success = success;
       this.done = done;
@@ -112,313 +49,5 @@ public class SiteIndexer {
     }
   }
 
-  private final SchemaFactory<ReviewDb> schemaFactory;
+  Result indexAll(I index);
-  private final ChangeData.Factory changeDataFactory;
-  private final GitRepositoryManager repoManager;
-  private final ListeningExecutorService executor;
-  private final ChangeIndexer.Factory indexerFactory;
-  private final ChangeNotes.Factory notesFactory;
-  private final ThreeWayMergeStrategy mergeStrategy;
-
-  private int numChanges = -1;
-  private OutputStream progressOut = NullOutputStream.INSTANCE;
-  private PrintWriter verboseWriter =
-      new PrintWriter(NullOutputStream.INSTANCE);
-
-  @Inject
-  SiteIndexer(SchemaFactory<ReviewDb> schemaFactory,
-      ChangeData.Factory changeDataFactory,
-      GitRepositoryManager repoManager,
-      @IndexExecutor(BATCH) ListeningExecutorService executor,
-      ChangeIndexer.Factory indexerFactory,
-      ChangeNotes.Factory notesFactory,
-      @GerritServerConfig Config config) {
-    this.schemaFactory = schemaFactory;
-    this.changeDataFactory = changeDataFactory;
-    this.repoManager = repoManager;
-    this.executor = executor;
-    this.indexerFactory = indexerFactory;
-    this.notesFactory = notesFactory;
-    this.mergeStrategy = MergeUtil.getMergeStrategy(config);
-  }
-
-  public SiteIndexer setNumChanges(int num) {
-    numChanges = num;
-    return this;
-  }
-
-  public SiteIndexer setProgressOut(OutputStream out) {
-    progressOut = checkNotNull(out);
-    return this;
-  }
-
-  public SiteIndexer setVerboseOut(OutputStream out) {
-    verboseWriter = new PrintWriter(checkNotNull(out));
-    return this;
-  }
-
-  public Result indexAll(ChangeIndex index,
-      Iterable<Project.NameKey> projects) {
-    Stopwatch sw = Stopwatch.createStarted();
-    final MultiProgressMonitor mpm =
-        new MultiProgressMonitor(progressOut, "Reindexing changes");
-    final Task projTask = mpm.beginSubTask("projects",
-        (projects instanceof Collection)
-          ? ((Collection<?>) projects).size()
-          : MultiProgressMonitor.UNKNOWN);
-    final Task doneTask = mpm.beginSubTask(null,
-        numChanges >= 0 ? numChanges : MultiProgressMonitor.UNKNOWN);
-    final Task failedTask = mpm.beginSubTask("failed", MultiProgressMonitor.UNKNOWN);
-
-    final List<ListenableFuture<?>> futures = Lists.newArrayList();
-    final AtomicBoolean ok = new AtomicBoolean(true);
-
-    for (final Project.NameKey project : projects) {
-      final ListenableFuture<?> future = executor.submit(reindexProject(
-          indexerFactory.create(executor, index), project, doneTask, failedTask,
-          verboseWriter));
-      futures.add(future);
-      future.addListener(new Runnable() {
-        @Override
-        public void run() {
-          try {
-            future.get();
-          } catch (ExecutionException | InterruptedException e) {
-            fail(project, e);
-          } catch (RuntimeException e) {
-            failAndThrow(project, e);
-          } catch (Error e) {
-            // Can't join with RuntimeException because "RuntimeException |
-            // Error" becomes Throwable, which messes with signatures.
-            failAndThrow(project, e);
-          } finally {
-            projTask.update(1);
-          }
-        }
-
-        private void fail(Project.NameKey project, Throwable t) {
-          log.error("Failed to index project " + project, t);
-          ok.set(false);
-        }
-
-        private void failAndThrow(Project.NameKey project, RuntimeException e) {
-          fail(project, e);
-          throw e;
-        }
-
-        private void failAndThrow(Project.NameKey project, Error e) {
-          fail(project, e);
-          throw e;
-        }
-      }, MoreExecutors.directExecutor());
-    }
-
-    try {
-      mpm.waitFor(Futures.transformAsync(Futures.successfulAsList(futures),
-          new AsyncFunction<List<?>, Void>() {
-            @Override
-            public ListenableFuture<Void> apply(List<?> input) {
-              mpm.end();
-              return Futures.immediateFuture(null);
-            }
-      }));
-    } catch (ExecutionException e) {
-      log.error("Error in batch indexer", e);
-      ok.set(false);
-    }
-    // If too many changes failed, maybe there was a bug in the indexer. Don't
-    // trust the results. This is not an exact percentage since we bump the same
-    // failure counter if a project can't be read, but close enough.
-    int nFailed = failedTask.getCount();
-    int nTotal = nFailed + doneTask.getCount();
-    double pctFailed = ((double) nFailed) / nTotal * 100;
-    if (pctFailed > 10) {
-      log.error("Failed {}/{} changes ({}%); not marking new index as ready",
-          nFailed, nTotal, Math.round(pctFailed));
-      ok.set(false);
-    }
-    return new Result(sw, ok.get(), doneTask.getCount(), failedTask.getCount());
-  }
-
-  private Callable<Void> reindexProject(final ChangeIndexer indexer,
-      final Project.NameKey project, final Task done, final Task failed,
-      final PrintWriter verboseWriter) {
-    return new Callable<Void>() {
-      @Override
-      public Void call() throws Exception {
-        Multimap<ObjectId, ChangeData> byId = ArrayListMultimap.create();
-        // TODO(dborowitz): Opening all repositories in a live server may be
-        // wasteful; see if we can determine which ones it is safe to close
-        // with RepositoryCache.close(repo).
-        try (Repository repo = repoManager.openRepository(project);
-            ReviewDb db = schemaFactory.open()) {
-          Map<String, Ref> refs = repo.getRefDatabase().getRefs(ALL);
-          for (ChangeNotes cn : notesFactory.scan(repo, db, project)) {
-            Ref r = refs.get(cn.getChange().currentPatchSetId().toRefName());
-            if (r != null) {
-              byId.put(r.getObjectId(), changeDataFactory.create(db, cn));
-            }
-          }
-          new ProjectIndexer(indexer,
-              mergeStrategy,
-              byId,
-              repo,
-              done,
-              failed,
-              verboseWriter).call();
-        } catch (RepositoryNotFoundException rnfe) {
-          log.error(rnfe.getMessage());
-        }
-        return null;
-      }
-
-      @Override
-      public String toString() {
-        return "Index all changes of project " + project.get();
-      }
-    };
-  }
-
-  private static class ProjectIndexer implements Callable<Void> {
-    private final ChangeIndexer indexer;
-    private final ThreeWayMergeStrategy mergeStrategy;
-    private final Multimap<ObjectId, ChangeData> byId;
-    private final ProgressMonitor done;
-    private final ProgressMonitor failed;
-    private final PrintWriter verboseWriter;
-    private final Repository repo;
-    private RevWalk walk;
-
-    private ProjectIndexer(ChangeIndexer indexer,
-        ThreeWayMergeStrategy mergeStrategy,
-        Multimap<ObjectId, ChangeData> changesByCommitId,
-        Repository repo,
-        ProgressMonitor done,
-        ProgressMonitor failed,
-        PrintWriter verboseWriter) {
-      this.indexer = indexer;
-      this.mergeStrategy = mergeStrategy;
-      this.byId = changesByCommitId;
-      this.repo = repo;
-      this.done = done;
-      this.failed = failed;
-      this.verboseWriter = verboseWriter;
-    }
-
-    @Override
-    public Void call() throws Exception {
-      walk = new RevWalk(repo);
-      try {
-        // Walk only refs first to cover as many changes as we can without having
-        // to mark every single change.
-        for (Ref ref : repo.getRefDatabase().getRefs(Constants.R_HEADS).values()) {
-          RevObject o = walk.parseAny(ref.getObjectId());
-          if (o instanceof RevCommit) {
-            walk.markStart((RevCommit) o);
-          }
-        }
-
-        RevCommit bCommit;
-        while ((bCommit = walk.next()) != null && !byId.isEmpty()) {
-          if (byId.containsKey(bCommit)) {
-            getPathsAndIndex(bCommit);
-            byId.removeAll(bCommit);
-          }
-        }
-
-        for (ObjectId id : byId.keySet()) {
-          getPathsAndIndex(id);
-        }
-      } finally {
-        walk.close();
-      }
-      return null;
-    }
-
-    private void getPathsAndIndex(ObjectId b) throws Exception {
-      List<ChangeData> cds = Lists.newArrayList(byId.get(b));
-      try (DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE)) {
-        RevCommit bCommit = walk.parseCommit(b);
-        RevTree bTree = bCommit.getTree();
-        RevTree aTree = aFor(bCommit, walk);
-        df.setRepository(repo);
-        if (!cds.isEmpty()) {
-          List<String> paths = (aTree != null)
-              ? getPaths(df.scan(aTree, bTree))
-              : Collections.<String>emptyList();
-          Iterator<ChangeData> cdit = cds.iterator();
-          for (ChangeData cd ; cdit.hasNext(); cdit.remove()) {
-            cd = cdit.next();
-            try {
-              cd.setCurrentFilePaths(paths);
-              indexer.index(cd);
-              done.update(1);
-              if (verboseWriter != null) {
-                verboseWriter.println("Reindexed change " + cd.getId());
-              }
-            } catch (Exception e) {
-              fail("Failed to index change " + cd.getId(), true, e);
-            }
-          }
-        }
-      } catch (Exception e) {
-        fail("Failed to index commit " + b.name(), false, e);
-        for (ChangeData cd : cds) {
-          fail("Failed to index change " + cd.getId(), true, null);
-        }
-      }
-    }
-
-    private List<String> getPaths(List<DiffEntry> filenames) {
-      Set<String> paths = Sets.newTreeSet();
-      for (DiffEntry e : filenames) {
-        if (e.getOldPath() != null) {
-          paths.add(e.getOldPath());
-        }
-        if (e.getNewPath() != null) {
-          paths.add(e.getNewPath());
-        }
-      }
-      return ImmutableList.copyOf(paths);
-    }
-
-    private RevTree aFor(RevCommit b, RevWalk walk) throws IOException {
-      switch (b.getParentCount()) {
-        case 0:
-          return walk.parseTree(emptyTree());
-        case 1:
-          RevCommit a = b.getParent(0);
-          walk.parseBody(a);
-          return walk.parseTree(a.getTree());
-        case 2:
-          return PatchListLoader.automerge(repo, walk, b, mergeStrategy);
-        default:
-          return null;
-      }
-    }
-
-    private ObjectId emptyTree() throws IOException {
-      try (ObjectInserter oi = repo.newObjectInserter()) {
-        ObjectId id = oi.insert(Constants.OBJ_TREE, new byte[] {});
-        oi.flush();
-        return id;
-      }
-    }
-
-    private void fail(String error, boolean failed, Exception e) {
-      if (failed) {
-        this.failed.update(1);
-      }
-
-      if (e != null) {
-        log.warn(error, e);
-      } else {
-        log.warn(error);
-      }
-
-      if (verboseWriter != null) {
-        verboseWriter.println(error);
-      }
-    }
-  }
 }

gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountField.java

@@ -0,0 +1,140 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index.account;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import com.google.gerrit.reviewdb.client.AccountExternalId;
+import com.google.gerrit.server.account.AccountState;
+import com.google.gerrit.server.index.FieldDef;
+import com.google.gerrit.server.index.FieldType;
+import com.google.gerrit.server.index.SchemaUtil;
+
+import java.sql.Timestamp;
+import java.util.Collections;
+import java.util.Set;
+
+/** Secondary index schemas for accounts. */
+public class AccountField {
+  public static final FieldDef<AccountState, Integer> ID =
+      new FieldDef.Single<AccountState, Integer>(
+          "id", FieldType.INTEGER, true) {
+        @Override
+        public Integer get(AccountState input, FillArgs args) {
+          return input.getAccount().getId().get();
+        }
+      };
+
+  public static final FieldDef<AccountState, Iterable<String>> EXTERNAL_ID =
+      new FieldDef.Repeatable<AccountState, String>(
+          "external_id", FieldType.EXACT, false) {
+        @Override
+        public Iterable<String> get(AccountState input, FillArgs args) {
+          return Iterables.transform(
+              input.getExternalIds(),
+              new Function<AccountExternalId, String>() {
+                @Override
+                public String apply(AccountExternalId in) {
+                  return in.getKey().get();
+                }
+              });
+        }
+      };
+
+  /** Fuzzy prefix match on name and email parts. */
+  public static final FieldDef<AccountState, Iterable<String>> NAME_PART =
+      new FieldDef.Repeatable<AccountState, String>(
+          "name", FieldType.PREFIX, false) {
+        @Override
+        public Iterable<String> get(AccountState input, FillArgs args) {
+          String fullName = input.getAccount().getFullName();
+          Set<String> parts = SchemaUtil.getPersonParts(
+              fullName,
+              Iterables.transform(
+                  input.getExternalIds(),
+                  new Function<AccountExternalId, String>() {
+                    @Override
+                    public String apply(AccountExternalId in) {
+                      return in.getEmailAddress();
+                    }
+                  }));
+
+          // Additional values not currently added by getPersonParts.
+          // TODO(dborowitz): Move to getPersonParts and remove this hack.
+          if (fullName != null) {
+            parts.add(fullName);
+          }
+          return parts;
+        }
+      };
+
+  public static final FieldDef<AccountState, String> ACTIVE =
+      new FieldDef.Single<AccountState, String>(
+          "inactive", FieldType.EXACT, false) {
+        @Override
+        public String get(AccountState input, FillArgs args) {
+          return input.getAccount().isActive() ? "1" : "0";
+        }
+      };
+
+  public static final FieldDef<AccountState, Iterable<String>> EMAIL =
+      new FieldDef.Repeatable<AccountState, String>(
+          "email", FieldType.PREFIX, false) {
+        @Override
+        public Iterable<String> get(AccountState input, FillArgs args) {
+          return FluentIterable.from(input.getExternalIds())
+            .transform(
+                new Function<AccountExternalId, String>() {
+                  @Override
+                  public String apply(AccountExternalId in) {
+                    return in.getEmailAddress();
+                  }
+                })
+            .append(
+                Collections.singleton(input.getAccount().getPreferredEmail()))
+            .filter(Predicates.notNull())
+            .transform(
+                new Function<String, String>() {
+                  @Override
+                  public String apply(String in) {
+                    return in.toLowerCase();
+                  }
+                });
+        }
+      };
+
+  public static final FieldDef<AccountState, Timestamp> REGISTERED =
+      new FieldDef.Single<AccountState, Timestamp>(
+          "registered", FieldType.TIMESTAMP, false) {
+        @Override
+        public Timestamp get(AccountState input, FillArgs args) {
+          return input.getAccount().getRegisteredOn();
+        }
+      };
+
+  public static final FieldDef<AccountState, String> USERNAME =
+      new FieldDef.Single<AccountState, String>(
+            "username", null, false) {
+        @Override
+        public String get(AccountState input, FillArgs args) {
+          return input.getUserName().toLowerCase();
+        }
+      };
+
+  private AccountField() {
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/index/account/AccountSchemas.java

@@ -0,0 +1,62 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index.account;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.gerrit.server.account.AccountState;
+import com.google.gerrit.server.index.FieldDef;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.index.SchemaUtil;
+
+import java.util.Collection;
+
+public class AccountSchemas {
+  static final Schema<AccountState> V1 = schema(
+      AccountField.ID,
+      AccountField.ACTIVE,
+      AccountField.EMAIL,
+      AccountField.EXTERNAL_ID,
+      AccountField.NAME_PART,
+      AccountField.REGISTERED,
+      AccountField.USERNAME);
+
+  private static Schema<AccountState> schema(
+      Collection<FieldDef<AccountState, ?>> fields) {
+    return new Schema<>(ImmutableList.copyOf(fields));
+  }
+
+  @SafeVarargs
+  private static Schema<AccountState> schema(
+      FieldDef<AccountState, ?>... fields) {
+    return schema(ImmutableList.copyOf(fields));
+  }
+
+  public static final ImmutableMap<Integer, Schema<AccountState>> ALL =
+      SchemaUtil.schemasFromClass(AccountSchemas.class, AccountState.class);
+
+  public static Schema<AccountState> get(int version) {
+    Schema<AccountState> schema = ALL.get(version);
+    checkArgument(schema != null, "Unrecognized schema version: %s", version);
+    return schema;
+  }
+
+  public static Schema<AccountState> getLatest() {
+    return Iterables.getLast(ALL.values());
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/index/change/AllChangesIndexer.java

@@ -0,0 +1,404 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index.change;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;
+import static org.eclipse.jgit.lib.RefDatabase.ALL;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.gerrit.reviewdb.client.Change;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.reviewdb.server.ReviewDb;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.gerrit.server.git.MergeUtil;
+import com.google.gerrit.server.git.MultiProgressMonitor;
+import com.google.gerrit.server.git.MultiProgressMonitor.Task;
+import com.google.gerrit.server.index.IndexExecutor;
+import com.google.gerrit.server.index.SiteIndexer;
+import com.google.gerrit.server.notedb.ChangeNotes;
+import com.google.gerrit.server.patch.PatchListLoader;
+import com.google.gerrit.server.project.ProjectCache;
+import com.google.gerrit.server.query.change.ChangeData;
+import com.google.gwtorm.server.SchemaFactory;
+import com.google.inject.Inject;
+
+import org.eclipse.jgit.diff.DiffEntry;
+import org.eclipse.jgit.diff.DiffFormatter;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectInserter;
+import org.eclipse.jgit.lib.ProgressMonitor;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.merge.ThreeWayMergeStrategy;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevObject;
+import org.eclipse.jgit.revwalk.RevTree;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.util.io.DisabledOutputStream;
+import org.eclipse.jgit.util.io.NullOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class AllChangesIndexer
+    implements SiteIndexer<Change.Id, ChangeData, ChangeIndex> {
+  private static final Logger log =
+      LoggerFactory.getLogger(AllChangesIndexer.class);
+
+  private final SchemaFactory<ReviewDb> schemaFactory;
+  private final ChangeData.Factory changeDataFactory;
+  private final GitRepositoryManager repoManager;
+  private final ListeningExecutorService executor;
+  private final ChangeIndexer.Factory indexerFactory;
+  private final ChangeNotes.Factory notesFactory;
+  private final ProjectCache projectCache;
+  private final ThreeWayMergeStrategy mergeStrategy;
+
+  private int totalWork = -1;
+  private OutputStream progressOut = NullOutputStream.INSTANCE;
+  private PrintWriter verboseWriter =
+      new PrintWriter(NullOutputStream.INSTANCE);
+
+  @Inject
+  AllChangesIndexer(SchemaFactory<ReviewDb> schemaFactory,
+      ChangeData.Factory changeDataFactory,
+      GitRepositoryManager repoManager,
+      @IndexExecutor(BATCH) ListeningExecutorService executor,
+      ChangeIndexer.Factory indexerFactory,
+      ChangeNotes.Factory notesFactory,
+      @GerritServerConfig Config config,
+      ProjectCache projectCache) {
+    this.schemaFactory = schemaFactory;
+    this.changeDataFactory = changeDataFactory;
+    this.repoManager = repoManager;
+    this.executor = executor;
+    this.indexerFactory = indexerFactory;
+    this.notesFactory = notesFactory;
+    this.projectCache = projectCache;
+    this.mergeStrategy = MergeUtil.getMergeStrategy(config);
+  }
+
+  public AllChangesIndexer setTotalWork(int num) {
+    totalWork = num;
+    return this;
+  }
+
+  public AllChangesIndexer setProgressOut(OutputStream out) {
+    progressOut = checkNotNull(out);
+    return this;
+  }
+
+  public AllChangesIndexer setVerboseOut(OutputStream out) {
+    verboseWriter = new PrintWriter(checkNotNull(out));
+    return this;
+  }
+
+  @Override
+  public Result indexAll(ChangeIndex index) {
+    return indexAll(index, projectCache.all());
+  }
+
+  public SiteIndexer.Result indexAll(ChangeIndex index,
+      Iterable<Project.NameKey> projects) {
+    Stopwatch sw = Stopwatch.createStarted();
+    final MultiProgressMonitor mpm =
+        new MultiProgressMonitor(progressOut, "Reindexing changes");
+    final Task projTask = mpm.beginSubTask("projects",
+        (projects instanceof Collection)
+          ? ((Collection<?>) projects).size()
+          : MultiProgressMonitor.UNKNOWN);
+    final Task doneTask = mpm.beginSubTask(null,
+        totalWork >= 0 ? totalWork : MultiProgressMonitor.UNKNOWN);
+    final Task failedTask = mpm.beginSubTask("failed", MultiProgressMonitor.UNKNOWN);
+
+    final List<ListenableFuture<?>> futures = Lists.newArrayList();
+    final AtomicBoolean ok = new AtomicBoolean(true);
+
+    for (final Project.NameKey project : projects) {
+      final ListenableFuture<?> future = executor.submit(reindexProject(
+          indexerFactory.create(executor, index), project, doneTask, failedTask,
+          verboseWriter));
+      futures.add(future);
+      future.addListener(new Runnable() {
+        @Override
+        public void run() {
+          try {
+            future.get();
+          } catch (ExecutionException | InterruptedException e) {
+            fail(project, e);
+          } catch (RuntimeException e) {
+            failAndThrow(project, e);
+          } catch (Error e) {
+            // Can't join with RuntimeException because "RuntimeException |
+            // Error" becomes Throwable, which messes with signatures.
+            failAndThrow(project, e);
+          } finally {
+            projTask.update(1);
+          }
+        }
+
+        private void fail(Project.NameKey project, Throwable t) {
+          log.error("Failed to index project " + project, t);
+          ok.set(false);
+        }
+
+        private void failAndThrow(Project.NameKey project, RuntimeException e) {
+          fail(project, e);
+          throw e;
+        }
+
+        private void failAndThrow(Project.NameKey project, Error e) {
+          fail(project, e);
+          throw e;
+        }
+      }, MoreExecutors.directExecutor());
+    }
+
+    try {
+      mpm.waitFor(Futures.transformAsync(Futures.successfulAsList(futures),
+          new AsyncFunction<List<?>, Void>() {
+            @Override
+            public ListenableFuture<Void> apply(List<?> input) {
+              mpm.end();
+              return Futures.immediateFuture(null);
+            }
+      }));
+    } catch (ExecutionException e) {
+      log.error("Error in batch indexer", e);
+      ok.set(false);
+    }
+    // If too many changes failed, maybe there was a bug in the indexer. Don't
+    // trust the results. This is not an exact percentage since we bump the same
+    // failure counter if a project can't be read, but close enough.
+    int nFailed = failedTask.getCount();
+    int nTotal = nFailed + doneTask.getCount();
+    double pctFailed = ((double) nFailed) / nTotal * 100;
+    if (pctFailed > 10) {
+      log.error("Failed {}/{} changes ({}%); not marking new index as ready",
+          nFailed, nTotal, Math.round(pctFailed));
+      ok.set(false);
+    }
+    return new Result(sw, ok.get(), doneTask.getCount(), failedTask.getCount());
+  }
+
+  private Callable<Void> reindexProject(final ChangeIndexer indexer,
+      final Project.NameKey project, final Task done, final Task failed,
+      final PrintWriter verboseWriter) {
+    return new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        Multimap<ObjectId, ChangeData> byId = ArrayListMultimap.create();
+        // TODO(dborowitz): Opening all repositories in a live server may be
+        // wasteful; see if we can determine which ones it is safe to close
+        // with RepositoryCache.close(repo).
+        try (Repository repo = repoManager.openRepository(project);
+            ReviewDb db = schemaFactory.open()) {
+          Map<String, Ref> refs = repo.getRefDatabase().getRefs(ALL);
+          for (ChangeNotes cn : notesFactory.scan(repo, db, project)) {
+            Ref r = refs.get(cn.getChange().currentPatchSetId().toRefName());
+            if (r != null) {
+              byId.put(r.getObjectId(), changeDataFactory.create(db, cn));
+            }
+          }
+          new ProjectIndexer(indexer,
+              mergeStrategy,
+              byId,
+              repo,
+              done,
+              failed,
+              verboseWriter).call();
+        } catch (RepositoryNotFoundException rnfe) {
+          log.error(rnfe.getMessage());
+        }
+        return null;
+      }
+
+      @Override
+      public String toString() {
+        return "Index all changes of project " + project.get();
+      }
+    };
+  }
+
+  private static class ProjectIndexer implements Callable<Void> {
+    private final ChangeIndexer indexer;
+    private final ThreeWayMergeStrategy mergeStrategy;
+    private final Multimap<ObjectId, ChangeData> byId;
+    private final ProgressMonitor done;
+    private final ProgressMonitor failed;
+    private final PrintWriter verboseWriter;
+    private final Repository repo;
+    private RevWalk walk;
+
+    private ProjectIndexer(ChangeIndexer indexer,
+        ThreeWayMergeStrategy mergeStrategy,
+        Multimap<ObjectId, ChangeData> changesByCommitId,
+        Repository repo,
+        ProgressMonitor done,
+        ProgressMonitor failed,
+        PrintWriter verboseWriter) {
+      this.indexer = indexer;
+      this.mergeStrategy = mergeStrategy;
+      this.byId = changesByCommitId;
+      this.repo = repo;
+      this.done = done;
+      this.failed = failed;
+      this.verboseWriter = verboseWriter;
+    }
+
+    @Override
+    public Void call() throws Exception {
+      walk = new RevWalk(repo);
+      try {
+        // Walk only refs first to cover as many changes as we can without having
+        // to mark every single change.
+        for (Ref ref : repo.getRefDatabase().getRefs(Constants.R_HEADS).values()) {
+          RevObject o = walk.parseAny(ref.getObjectId());
+          if (o instanceof RevCommit) {
+            walk.markStart((RevCommit) o);
+          }
+        }
+
+        RevCommit bCommit;
+        while ((bCommit = walk.next()) != null && !byId.isEmpty()) {
+          if (byId.containsKey(bCommit)) {
+            getPathsAndIndex(bCommit);
+            byId.removeAll(bCommit);
+          }
+        }
+
+        for (ObjectId id : byId.keySet()) {
+          getPathsAndIndex(id);
+        }
+      } finally {
+        walk.close();
+      }
+      return null;
+    }
+
+    private void getPathsAndIndex(ObjectId b) throws Exception {
+      List<ChangeData> cds = Lists.newArrayList(byId.get(b));
+      try (DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE)) {
+        RevCommit bCommit = walk.parseCommit(b);
+        RevTree bTree = bCommit.getTree();
+        RevTree aTree = aFor(bCommit, walk);
+        df.setRepository(repo);
+        if (!cds.isEmpty()) {
+          List<String> paths = (aTree != null)
+              ? getPaths(df.scan(aTree, bTree))
+              : Collections.<String>emptyList();
+          Iterator<ChangeData> cdit = cds.iterator();
+          for (ChangeData cd ; cdit.hasNext(); cdit.remove()) {
+            cd = cdit.next();
+            try {
+              cd.setCurrentFilePaths(paths);
+              indexer.index(cd);
+              done.update(1);
+              if (verboseWriter != null) {
+                verboseWriter.println("Reindexed change " + cd.getId());
+              }
+            } catch (Exception e) {
+              fail("Failed to index change " + cd.getId(), true, e);
+            }
+          }
+        }
+      } catch (Exception e) {
+        fail("Failed to index commit " + b.name(), false, e);
+        for (ChangeData cd : cds) {
+          fail("Failed to index change " + cd.getId(), true, null);
+        }
+      }
+    }
+
+    private List<String> getPaths(List<DiffEntry> filenames) {
+      Set<String> paths = Sets.newTreeSet();
+      for (DiffEntry e : filenames) {
+        if (e.getOldPath() != null) {
+          paths.add(e.getOldPath());
+        }
+        if (e.getNewPath() != null) {
+          paths.add(e.getNewPath());
+        }
+      }
+      return ImmutableList.copyOf(paths);
+    }
+
+    private RevTree aFor(RevCommit b, RevWalk walk) throws IOException {
+      switch (b.getParentCount()) {
+        case 0:
+          return walk.parseTree(emptyTree());
+        case 1:
+          RevCommit a = b.getParent(0);
+          walk.parseBody(a);
+          return walk.parseTree(a.getTree());
+        case 2:
+          return PatchListLoader.automerge(repo, walk, b, mergeStrategy);
+        default:
+          return null;
+      }
+    }
+
+    private ObjectId emptyTree() throws IOException {
+      try (ObjectInserter oi = repo.newObjectInserter()) {
+        ObjectId id = oi.insert(Constants.OBJ_TREE, new byte[] {});
+        oi.flush();
+        return id;
+      }
+    }
+
+    private void fail(String error, boolean failed, Exception e) {
+      if (failed) {
+        this.failed.update(1);
+      }
+
+      if (e != null) {
+        log.warn(error, e);
+      } else {
+        log.warn(error);
+      }
+
+      if (verboseWriter != null) {
+        verboseWriter.println(error);
+      }
+    }
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndex.java

@@ -16,7 +16,11 @@ package com.google.gerrit.server.index.change;
 
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.server.index.Index;
+import com.google.gerrit.server.index.IndexDefinition;
 import com.google.gerrit.server.query.change.ChangeData;
 
 public interface ChangeIndex extends Index<Change.Id, ChangeData> {
+  public interface Factory extends
+      IndexDefinition.IndexFactory<Change.Id, ChangeData, ChangeIndex> {
+  }
 }

gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeIndexDefintion.java

@@ -0,0 +1,33 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.index.change;
+
+import com.google.gerrit.reviewdb.client.Change;
+import com.google.gerrit.server.index.IndexDefinition;
+import com.google.gerrit.server.query.change.ChangeData;
+import com.google.inject.Inject;
+
+public class ChangeIndexDefintion
+    extends IndexDefinition<Change.Id, ChangeData, ChangeIndex> {
+
+  @Inject
+  ChangeIndexDefintion(
+      ChangeIndexCollection indexCollection,
+      ChangeIndex.Factory indexFactory,
+      AllChangesIndexer allChangesIndexer) {
+    super(ChangeSchemaDefinitions.INSTANCE, indexCollection, indexFactory,
+        allChangesIndexer);
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemaDefinitions.java

@@ -1,4 +1,4 @@
-// Copyright (C) 2013 The Android Open Source Project
+// Copyright (C) 2016 The Android Open Source Project
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,17 +14,13 @@
 
 package com.google.gerrit.server.index.change;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.gerrit.server.index.SchemaUtil.schema;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
 import com.google.gerrit.server.index.Schema;
-import com.google.gerrit.server.index.SchemaUtil;
+import com.google.gerrit.server.index.SchemaDefinitions;
 import com.google.gerrit.server.query.change.ChangeData;
 
-/** Secondary index schemas for changes. */
+public class ChangeSchemaDefinitions extends SchemaDefinitions<ChangeData> {
-public class ChangeSchemas {
   @Deprecated
   static final Schema<ChangeData> V25 = schema(
       ChangeField.LEGACY_ID,
@@ -102,16 +98,10 @@ public class ChangeSchemas {
 
   static final Schema<ChangeData> V27 = schema(V26.getFields().values());
 
-  public static final ImmutableMap<Integer, Schema<ChangeData>> ALL =
+  public static final ChangeSchemaDefinitions INSTANCE =
-      SchemaUtil.schemasFromClass(ChangeSchemas.class, ChangeData.class);
+      new ChangeSchemaDefinitions();
 
-  public static Schema<ChangeData> get(int version) {
+  private ChangeSchemaDefinitions() {
-    Schema<ChangeData> schema = ALL.get(version);
+    super("changes", ChangeData.class);
-    checkArgument(schema != null, "Unrecognized schema version: %s", version);
-    return schema;
-  }
-
-  public static Schema<ChangeData> getLatest() {
-    return Iterables.getLast(ALL.values());
   }
 }

gerrit-server/src/test/java/com/google/gerrit/testutil/InMemoryModule.java

@@ -14,7 +14,6 @@
 
 package com.google.gerrit.testutil;
 
-import static com.google.common.base.Preconditions.checkState;
 import static com.google.inject.Scopes.SINGLETON;
 
 import com.google.common.util.concurrent.MoreExecutors;
@@ -49,7 +48,7 @@ import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.git.PerThreadRequestScope;
 import com.google.gerrit.server.git.SendEmailExecutor;
 import com.google.gerrit.server.index.IndexModule.IndexType;
-import com.google.gerrit.server.index.change.ChangeSchemas;
+import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
 import com.google.gerrit.server.mail.SignedTokenEmailTokenVerifier;
 import com.google.gerrit.server.notedb.NotesMigration;
 import com.google.gerrit.server.patch.DiffExecutor;
@@ -74,10 +73,12 @@ import com.google.inject.servlet.RequestScoped;
 import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.lib.PersonIdent;
 
-import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.ExecutorService;
 
 public class InMemoryModule extends FactoryModule {
@@ -97,8 +98,6 @@ public class InMemoryModule extends FactoryModule {
     cfg.unset("cache", null, "directory");
     cfg.setString("index", null, "type", "lucene");
     cfg.setBoolean("index", "lucene", "testInmemory", true);
-    cfg.setInt("index", "lucene", "testVersion",
-        ChangeSchemas.getLatest().getVersion());
     cfg.setInt("sendemail", null, "threadPoolSize", 0);
     cfg.setBoolean("receive", null, "enableSignedPush", false);
     cfg.setString("receive", null, "certNonceSeed", "sekret");
@@ -234,16 +233,20 @@ public class InMemoryModule extends FactoryModule {
 
   private Module luceneIndexModule() {
     try {
+      Map<String, Integer> singleVersions = new HashMap<>();
       int version = cfg.getInt("index", "lucene", "testVersion", -1);
-      checkState(ChangeSchemas.ALL.containsKey(version),
+      if (version > 0) {
-          "invalid index.lucene.testVersion %s", version);
+        singleVersions.put(ChangeSchemaDefinitions.INSTANCE.getName(), version);
+      }
       Class<?> clazz =
           Class.forName("com.google.gerrit.lucene.LuceneIndexModule");
-      Constructor<?> c = clazz.getConstructor(Integer.class, int.class);
+      Method m = clazz.getMethod(
-      return (Module) c.newInstance(version, 0);
+          "singleVersionWithExplicitVersions", Map.class, int.class);
+      return (Module) m.invoke(null, singleVersions, 0);
     } catch (ClassNotFoundException | SecurityException | NoSuchMethodException
-        | IllegalArgumentException | InstantiationException
+        | IllegalArgumentException | IllegalAccessException
-        | IllegalAccessException | InvocationTargetException e) {
+        | InvocationTargetException e) {
+      e.printStackTrace();
       ProvisionException pe = new ProvisionException(e.getMessage());
       pe.initCause(e);
       throw pe;

gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java

@@ -337,7 +337,7 @@ public class WebAppInitializer extends GuiceServletContextListener
   private Module createIndexModule() {
     switch (indexType) {
       case LUCENE:
-        return new LuceneIndexModule();
+        return LuceneIndexModule.latestVersionWithOnlineUpgrade();
       default:
         throw new IllegalStateException("unsupported index.type = " + indexType);
     }