Update Lucene to 4.4.0

Of note in this version upgrade is the replacement of NrtManager by
ControlledRealTimeReopenThread, which has a different API as used by
SubIndex.

Change-Id: Id34eb8e75bb07a38be9b5ed9a1cdeaf195774b58

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java

@@ -14,15 +14,18 @@
 
 package com.google.gerrit.lucene;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES;
 import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES;
 
 import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.ListeningScheduledExecutorService;
 import com.google.gerrit.common.Nullable;
 import com.google.gerrit.reviewdb.client.Change;
@@ -33,6 +36,7 @@ import com.google.gerrit.server.index.ChangeField;
 import com.google.gerrit.server.index.ChangeField.ChangeProtoField;
 import com.google.gerrit.server.index.ChangeField.PatchSetApprovalProtoField;
 import com.google.gerrit.server.index.ChangeIndex;
+import com.google.gerrit.server.index.ChangeSchemas;
 import com.google.gerrit.server.index.FieldDef;
 import com.google.gerrit.server.index.FieldDef.FillArgs;
 import com.google.gerrit.server.index.FieldType;
@@ -84,9 +88,8 @@ import java.sql.Timestamp;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
 
 /**
  * Secondary index implementation using Apache Lucene.
@@ -100,20 +103,37 @@ public class LuceneChangeIndex implements ChangeIndex {
   private static final Logger log =
       LoggerFactory.getLogger(LuceneChangeIndex.class);
 
-  public static final Version LUCENE_VERSION = Version.LUCENE_43;
   public static final String CHANGES_OPEN = "open";
   public static final String CHANGES_CLOSED = "closed";
   private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
   private static final String CHANGE_FIELD = ChangeField.CHANGE.getName();
   private static final String APPROVAL_FIELD = ChangeField.APPROVAL.getName();
 
+  private static final Map<Schema<ChangeData>, Version> LUCENE_VERSIONS;
+  static {
+    ImmutableMap.Builder<Schema<ChangeData>, Version> versions =
+        ImmutableMap.builder();
+    @SuppressWarnings("deprecation")
+    Version lucene43 = Version.LUCENE_43;
+    for (Map.Entry<Integer, Schema<ChangeData>> e
+        : ChangeSchemas.ALL.entrySet()) {
+      if (e.getKey() <= 3) {
+        versions.put(e.getValue(), lucene43);
+      } else {
+        versions.put(e.getValue(), Version.LUCENE_44);
+      }
+    }
+    LUCENE_VERSIONS = versions.build();
+  }
+
   static interface Factory {
     LuceneChangeIndex create(Schema<ChangeData> schema, String base);
   }
 
-  private static IndexWriterConfig getIndexWriterConfig(Config cfg, String name) {
+  private static IndexWriterConfig getIndexWriterConfig(Version version,
-    IndexWriterConfig writerConfig = new IndexWriterConfig(LUCENE_VERSION,
+      Config cfg, String name) {
-        new StandardAnalyzer(LUCENE_VERSION, CharArraySet.EMPTY_SET));
+    IndexWriterConfig writerConfig = new IndexWriterConfig(version,
+        new StandardAnalyzer(version, CharArraySet.EMPTY_SET));
     writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
     double m = 1 << 20;
     writerConfig.setRAMBufferSizeMB(cfg.getLong("index", name, "ramBufferSize",
@@ -125,7 +145,7 @@ public class LuceneChangeIndex implements ChangeIndex {
 
   private final SitePaths sitePaths;
   private final FillArgs fillArgs;
-  private final ExecutorService executor;
+  private final ListeningExecutorService executor;
   private final File dir;
   private final Schema<ChangeData> schema;
   private final SubIndex openIndex;
@@ -149,15 +169,18 @@ public class LuceneChangeIndex implements ChangeIndex {
     } else {
       dir = new File(base);
     }
+    Version luceneVersion = checkNotNull(
+        LUCENE_VERSIONS.get(schema),
+        "unknown Lucene version for index schema: %s", schema);
     openIndex = new SubIndex(new File(dir, CHANGES_OPEN),
-        getIndexWriterConfig(cfg, "changes_open"));
+        getIndexWriterConfig(luceneVersion, cfg, "changes_open"));
     closedIndex = new SubIndex(new File(dir, CHANGES_CLOSED),
-        getIndexWriterConfig(cfg, "changes_closed"));
+        getIndexWriterConfig(luceneVersion, cfg, "changes_closed"));
   }
 
   @Override
   public void close() {
-    List<Future<?>> closeFutures = Lists.newArrayListWithCapacity(2);
+    List<ListenableFuture<?>> closeFutures = Lists.newArrayListWithCapacity(2);
     closeFutures.add(executor.submit(new Runnable() {
       @Override
       public void run() {
@@ -170,9 +193,7 @@ public class LuceneChangeIndex implements ChangeIndex {
         closedIndex.close();
       }
     }));
-    for (Future<?> future : closeFutures) {
+    Futures.getUnchecked(Futures.allAsList(closeFutures));
-      Futures.getUnchecked(future);
-    }
   }
 
   @Override

gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java

@@ -22,12 +22,12 @@ import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TrackingIndexWriter;
+import org.apache.lucene.search.ControlledRealTimeReopenThread;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NRTManager;
-import org.apache.lucene.search.NRTManager.TrackingIndexWriter;
-import org.apache.lucene.search.NRTManagerReopenThread;
 import org.apache.lucene.search.ReferenceManager.RefreshListener;
 import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.search.SearcherManager;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.slf4j.Logger;
@@ -48,17 +48,18 @@ class SubIndex {
 
   private final Directory dir;
   private final TrackingIndexWriter writer;
-  private final NRTManager nrtManager;
+  private final SearcherManager searcherManager;
-  private final NRTManagerReopenThread reopenThread;
+  private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
   private final ConcurrentMap<RefreshListener, Boolean> refreshListeners;
 
   SubIndex(File file, IndexWriterConfig writerConfig) throws IOException {
     dir = FSDirectory.open(file);
-    writer = new NRTManager.TrackingIndexWriter(new IndexWriter(dir, writerConfig));
+    writer = new TrackingIndexWriter(new IndexWriter(dir, writerConfig));
-    nrtManager = new NRTManager(writer, new SearcherFactory());
+    searcherManager = new SearcherManager(
+        writer.getIndexWriter(), true, new SearcherFactory());
 
     refreshListeners = Maps.newConcurrentMap();
-    nrtManager.addListener(new RefreshListener() {
+    searcherManager.addListener(new RefreshListener() {
       @Override
       public void beforeRefresh() throws IOException {
       }
@@ -71,8 +72,8 @@ class SubIndex {
       }
     });
 
-    reopenThread = new NRTManagerReopenThread(
+    reopenThread = new ControlledRealTimeReopenThread<IndexSearcher>(
-        nrtManager,
+        writer, searcherManager,
         0.500 /* maximum stale age (seconds) */,
         0.010 /* minimum stale age (seconds) */);
     reopenThread.setName("NRT " + file.getName());
@@ -86,12 +87,8 @@ class SubIndex {
   void close() {
     reopenThread.close();
     try {
-      nrtManager.close();
+      writer.getIndexWriter().commit();
-    } catch (IOException e) {
+      writer.getIndexWriter().close(true);
-      log.warn("error closing Lucene searcher", e);
-    }
-    try {
-      writer.getIndexWriter().close();
     } catch (IOException e) {
       log.warn("error closing Lucene writer", e);
     }
@@ -119,11 +116,11 @@ class SubIndex {
   }
 
   IndexSearcher acquire() throws IOException {
-    return nrtManager.acquire();
+    return searcherManager.acquire();
   }
 
   void release(IndexSearcher searcher) throws IOException {
-    nrtManager.release(searcher);
+    searcherManager.release(searcher);
   }
 
   private final class NrtFuture extends AbstractFuture<Void>
@@ -138,7 +135,7 @@ class SubIndex {
     @Override
     public Void get() throws InterruptedException, ExecutionException {
       if (!isDone()) {
-        nrtManager.waitForGeneration(gen);
+        reopenThread.waitForGeneration(gen);
         set(null);
       }
       return super.get();
@@ -148,7 +145,8 @@ class SubIndex {
     public Void get(long timeout, TimeUnit unit) throws InterruptedException,
         TimeoutException, ExecutionException {
       if (!isDone()) {
-        nrtManager.waitForGeneration(gen, timeout, unit);
+        reopenThread.waitForGeneration(gen,
+            (int) TimeUnit.MILLISECONDS.convert(timeout, unit));
         set(null);
       }
       return super.get(timeout, unit);
@@ -158,7 +156,7 @@ class SubIndex {
     public boolean isDone() {
       if (super.isDone()) {
         return true;
-      } else if (gen <= nrtManager.getCurrentSearchingGen()) {
+      } else if (isSearcherCurrent()) {
         set(null);
         return true;
       }
@@ -168,7 +166,7 @@ class SubIndex {
     @Override
     public void addListener(Runnable listener, Executor executor) {
       if (hasListeners.compareAndSet(false, true) && !isDone()) {
-        nrtManager.addListener(this);
+        searcherManager.addListener(this);
       }
       super.addListener(listener, executor);
     }
@@ -187,10 +185,19 @@ class SubIndex {
 
     @Override
     public void afterRefresh(boolean didRefresh) throws IOException {
-      if (gen <= nrtManager.getCurrentSearchingGen()) {
+      if (isSearcherCurrent()) {
         refreshListeners.remove(this);
         set(null);
       }
     }
+
+    private boolean isSearcherCurrent() {
+      try {
+        return reopenThread.waitForGeneration(gen, 0);
+      } catch (InterruptedException e) {
+        log.warn("Interrupted waiting for searcher generation", e);
+        return false;
+      }
+    }
   }
 }

gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java

@@ -25,6 +25,7 @@ import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.lang.reflect.ParameterizedType;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Map;
 
 /** Secondary index schemas for changes. */
@@ -93,8 +94,15 @@ public class ChangeSchemas {
         ChangeField.CHANGE,
         ChangeField.APPROVAL);
 
+  // For upgrade to Lucene 4.4.0 index format only.
+  static final Schema<ChangeData> V4 = release(V3.getFields().values());
+
+  private static Schema<ChangeData> release(Collection<FieldDef<ChangeData, ?>> fields) {
+    return new Schema<ChangeData>(true, fields);
+  }
+
   private static Schema<ChangeData> release(FieldDef<ChangeData, ?>... fields) {
-    return new Schema<ChangeData>(true, Arrays.asList(fields));
+    return release(Arrays.asList(fields));
   }
 
   @SuppressWarnings("unused")

gerrit-server/src/main/java/com/google/gerrit/server/index/Schema.java

@@ -15,6 +15,7 @@
 package com.google.gerrit.server.index;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
 import com.google.common.collect.ImmutableMap;
 
 /** Specific version of a secondary index schema. */
@@ -51,6 +52,13 @@ public class Schema<T> {
     return fields;
   }
 
+  @Override
+  public String toString() {
+    return Objects.toStringHelper(this)
+        .addValue(fields.keySet())
+        .toString();
+  }
+
   void setVersion(int version) {
     this.version = version;
   }

lib/lucene/BUCK

@@ -2,9 +2,9 @@ include_defs('//lib/maven.defs')
 
 maven_jar(
   name = 'core',
-  id = 'org.apache.lucene:lucene-core:4.3.0',
+  id = 'org.apache.lucene:lucene-core:4.4.0',
-  bin_sha1 = 'd4e40fe5661b8de5d8c66db3d63a47b6b3ecf7f3',
+  bin_sha1 = 'a9a0b553d5f2444aea3340b22753ea4bbddaa0af',
-  src_sha1 = '86c29288b1930e33ba7ffea1b866af9a52d3d24a',
+  src_sha1 = 'd321e15f688066a3c3598607303e0de452a076da',
   license = 'Apache2.0',
   exclude = [
     'META-INF/LICENSE.txt',
@@ -14,9 +14,9 @@ maven_jar(
 
 maven_jar(
   name = 'analyzers-common',
-  id = 'org.apache.lucene:lucene-analyzers-common:4.3.0',
+  id = 'org.apache.lucene:lucene-analyzers-common:4.4.0',
-  bin_sha1 = 'e7c3976156d292f696016e138b67ab5e6bfc1a56',
+  bin_sha1 = 'f58f6b727293b2d4392064db8c91fdf1d0eb4ffe',
-  src_sha1 = '3606622b3c1f09b4b7cf34070cbf60d414af9b6b',
+  src_sha1 = '60176bb63009f41104b42656b20c81b66313e7b5',
   license = 'Apache2.0',
   exclude = [
     'META-INF/LICENSE.txt',
@@ -26,17 +26,17 @@ maven_jar(
 
 maven_jar(
   name = 'highlighter',
-  id = 'org.apache.lucene:lucene-highlighter:4.3.0',
+  id = 'org.apache.lucene:lucene-highlighter:4.4.0',
-  bin_sha1 = '9e6d60921e16a0d6b2e609c6a02a8b08cd7f643c',
+  bin_sha1 = 'c55f402683388c0a71a1dfaaff198873dfe5b1e4',
-  src_sha1 = '0ff70cae1a8fb7af29bf254d90e9885961deed5e',
+  src_sha1 = '3a99f84e4b6a8f74c34b2f2bd076c9b2b46fff2e',
   license = 'Apache2.0',
 )
 
 maven_jar(
   name = 'queries',
-  id = 'org.apache.lucene:lucene-queries:4.3.0',
+  id = 'org.apache.lucene:lucene-queries:4.4.0',
-  bin_sha1 = '68e01022bdf4f869b95362c9af964846e5d3cf2d',
+  bin_sha1 = 'c9010f4852345ba2a65163fdeb17b7b653e4a3c4',
-  src_sha1 = '3e3541c1b9f44c532ce88ab6a12216566c3399df',
+  src_sha1 = 'eefbcd43e66747a412a9f186d183d187405374b8',
   license = 'Apache2.0',
 )