Run all Lucene queries on background threads

Backport of d2d2d3907f to stable-2.12.

The NIOFSDirectory type does not handle being interrupted well, as an
interrupt delivered by an SSH connection closing will close the file
handles used to read the index.

Isolate the Lucene reader from the SSH threads by always running a
Lucene query on the interactive index thread pool.  This may reduce
latency for user dashboards on a multi-core system as each query for
the different sections can now run on separate threads and return
results when ready.

Bug: Issue 4400
Change-Id: I9fd97298d000e2b32371b9e0c454f7db3ef867ab

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java

@@ -23,6 +23,7 @@ import static com.google.gerrit.server.index.IndexRewriter.OPEN_STATUSES;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.MINUTES;
 
+import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
@@ -58,6 +59,7 @@ import com.google.gerrit.server.query.change.LegacyChangeIdPredicate;
 import com.google.gerrit.server.query.change.QueryOptions;
 import com.google.gwtorm.protobuf.ProtobufCodec;
 import com.google.gwtorm.server.OrmException;
+import com.google.gwtorm.server.OrmRuntimeException;
 import com.google.gwtorm.server.ResultSet;
 import com.google.inject.Provider;
 import com.google.inject.assistedinject.Assisted;
@@ -110,7 +112,9 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 
 /**
  * Secondary index implementation using Apache Lucene.
@@ -426,6 +430,20 @@ public class LuceneChangeIndex implements ChangeIndex {
 
     @Override
     public ResultSet<ChangeData> read() throws OrmException {
+      if (Thread.interrupted()) {
+        Thread.currentThread().interrupt();
+        throw new OrmException("interupted");
+      }
+      return new ChangeDataResults(
+          executor.submit(new Callable<List<Document>>() {
+            @Override
+            public List<Document> call() throws OrmException {
+              return doRead();
+            }
+          }));
+    }
+
+    private List<Document> doRead() throws OrmException {
       IndexSearcher[] searchers = new IndexSearcher[indexes.size()];
       try {
         int realLimit = opts.start() + opts.limit();
@@ -436,31 +454,14 @@ public class LuceneChangeIndex implements ChangeIndex {
         }
         TopDocs docs = TopDocs.merge(sort, realLimit, hits);
 
-        List<ChangeData> result =
+        List<Document> result =
             Lists.newArrayListWithCapacity(docs.scoreDocs.length);
         for (int i = opts.start(); i < docs.scoreDocs.length; i++) {
           ScoreDoc sd = docs.scoreDocs[i];
           Document doc = searchers[sd.shardIndex].doc(sd.doc, FIELDS);
-          result.add(toChangeData(doc));
+          result.add(doc);
         }
-
+        return result;
-        final List<ChangeData> r = Collections.unmodifiableList(result);
-        return new ResultSet<ChangeData>() {
-          @Override
-          public Iterator<ChangeData> iterator() {
-            return r.iterator();
-          }
-
-          @Override
-          public List<ChangeData> toList() {
-            return r;
-          }
-
-          @Override
-          public void close() {
-            // Do nothing.
-          }
-        };
       } catch (IOException e) {
         throw new OrmException(e);
       } finally {
@@ -477,6 +478,42 @@ public class LuceneChangeIndex implements ChangeIndex {
     }
   }
 
+  private class ChangeDataResults implements ResultSet<ChangeData> {
+    private final Future<List<Document>> future;
+
+    ChangeDataResults(Future<List<Document>> future) {
+      this.future = future;
+    }
+
+    @Override
+    public Iterator<ChangeData> iterator() {
+      return toList().iterator();
+    }
+
+    @Override
+    public List<ChangeData> toList() {
+      try {
+        List<Document> docs = future.get();
+        List<ChangeData> result = new ArrayList<>(docs.size());
+        String idFieldName = ChangeField.LEGACY_ID.getName();
+        for (Document doc : docs) {
+          result.add(toChangeData(doc));
+        }
+        return result;
+      } catch (InterruptedException e) {
+        close();
+        throw new OrmRuntimeException(e);
+      } catch (ExecutionException e) {
+        Throwables.propagateIfPossible(e.getCause());
+        throw new OrmRuntimeException(e.getCause());
+      }
+    }
+
+    @Override
+    public void close() {
+      future.cancel(false /* do not interrupt Lucene */);
+    }
+  }
   private ChangeData toChangeData(Document doc) {
     BytesRef cb = doc.getBinaryValue(CHANGE_FIELD);
     if (cb == null) {

gerrit-server/src/main/java/com/google/gerrit/server/query/change/QueryProcessor.java

@@ -29,6 +29,7 @@ import com.google.gerrit.server.project.ChangeControl;
 import com.google.gerrit.server.query.Predicate;
 import com.google.gerrit.server.query.QueryParseException;
 import com.google.gwtorm.server.OrmException;
+import com.google.gwtorm.server.OrmRuntimeException;
 import com.google.gwtorm.server.ResultSet;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
@@ -100,7 +101,12 @@ public class QueryProcessor {
    */
   public List<QueryResult> queryChanges(List<Predicate<ChangeData>> queries)
       throws OrmException, QueryParseException {
-    return queryChanges(null, queries);
+    try {
+      return queryChanges(null, queries);
+    } catch (OrmRuntimeException e) {
+      throw new OrmException(e.getMessage(), e);
+    }
+
   }
 
   static {