Fix online reindexing

ChangeBatchIndexer submits new tasks to an executor, so each task
needs its own ReviewDb connection.

Ignore Lucene's exception on double-closing an IndexWriter. Try to
double-close less by not closing a search index when it may still be
written to, but this is still racy.

Change-Id: I6ee2863a1fbfc93904d0f2cdd45901d90696267b

gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java

@@ -28,6 +28,7 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ReferenceManager.RefreshListener;
 import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.slf4j.Logger;
@@ -93,7 +94,11 @@ class SubIndex {
     reopenThread.close();
     try {
       writer.getIndexWriter().commit();
-      writer.getIndexWriter().close(true);
+      try {
+        writer.getIndexWriter().close(true);
+      } catch (AlreadyClosedException e) {
+        // Ignore.
+      }
     } catch (IOException e) {
       log.warn("error closing Lucene writer", e);
     }

gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeBatchIndexer.java

@@ -35,8 +35,8 @@ import com.google.gerrit.server.git.MultiProgressMonitor;
 import com.google.gerrit.server.git.MultiProgressMonitor.Task;
 import com.google.gerrit.server.patch.PatchListLoader;
 import com.google.gerrit.server.query.change.ChangeData;
+import com.google.gwtorm.server.SchemaFactory;
 import com.google.inject.Inject;
-import com.google.inject.Provider;
 
 import org.eclipse.jgit.diff.DiffEntry;
 import org.eclipse.jgit.diff.DiffFormatter;
@@ -104,19 +104,19 @@ public class ChangeBatchIndexer {
     }
   }
 
-  private final Provider<ReviewDb> db;
+  private final SchemaFactory<ReviewDb> schemaFactory;
   private final ChangeData.Factory changeDataFactory;
   private final GitRepositoryManager repoManager;
   private final ListeningExecutorService executor;
   private final ChangeIndexer.Factory indexerFactory;
 
   @Inject
-  ChangeBatchIndexer(Provider<ReviewDb> db,
+  ChangeBatchIndexer(SchemaFactory<ReviewDb> schemaFactory,
       ChangeData.Factory changeDataFactory,
       GitRepositoryManager repoManager,
       @IndexExecutor ListeningExecutorService executor,
       ChangeIndexer.Factory indexerFactory) {
-    this.db = db;
+    this.schemaFactory = schemaFactory;
     this.changeDataFactory = changeDataFactory;
     this.repoManager = repoManager;
     this.executor = executor;
@@ -208,20 +208,25 @@ public class ChangeBatchIndexer {
       public Void call() throws Exception {
         Multimap<ObjectId, ChangeData> byId = ArrayListMultimap.create();
         Repository repo = null;
+        ReviewDb db = null;
         try {
           repo = repoManager.openRepository(project);
           Map<String, Ref> refs = repo.getRefDatabase().getRefs(ALL);
-          for (Change c : db.get().changes().byProject(project)) {
+          db = schemaFactory.open();
+          for (Change c : db.changes().byProject(project)) {
             Ref r = refs.get(c.currentPatchSetId().toRefName());
             if (r != null) {
-              byId.put(r.getObjectId(), changeDataFactory.create(db.get(), c));
+              byId.put(r.getObjectId(), changeDataFactory.create(db, c));
             }
           }
           new ProjectIndexer(indexer, byId, repo, done, failed, verboseWriter)
               .call();
-        } catch(RepositoryNotFoundException rnfe) {
+        } catch (RepositoryNotFoundException rnfe) {
           log.error(rnfe.getMessage());
         } finally {
+          if (db != null) {
+            db.close();
+          }
           if (repo != null) {
             repo.close();
           }

gerrit-server/src/main/java/com/google/gerrit/server/index/IndexCollection.java

@@ -45,7 +45,7 @@ public class IndexCollection implements LifecycleListener {
 
   public void setSearchIndex(ChangeIndex index) {
     ChangeIndex old = searchIndex.getAndSet(index);
-    if (old != null && old != index) {
+    if (old != null && old != index && !writeIndexes.contains(old)) {
       old.close();
     }
   }