Fix race in Lucene shutdown

After a ControlledRealTimeReopenThread is stopped, any NrtFutures
depending on that thread will never finish; get() will hang forever.
This means if there are any indexing operations happening in the
WorkQueue after indexes are closed, they will never complete, causing
a hang during WorkQueue#stop().

Swap the order that lifecycle listeners are registered between the
index module and the work queue, so that the work queue is shut down
first, and the reopen threads don't shut down and leave stranded
tasks.

Change-Id: I005315336178234a974cde0ac2e51f1e04fbd999

gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java

@@ -243,6 +243,9 @@ public class SubIndex {
       } else if (isGenAvailableNowForCurrentSearcher()) {
         set(null);
         return true;
+      } else if (!reopenThread.isAlive()) {
+        setException(new IllegalStateException("NRT thread is dead"));
+        return true;
       }
       return false;
     }

gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java

@@ -333,6 +333,11 @@ public class Daemon extends SiteProgram {
     modules.add(SchemaVersionCheck.module());
     modules.add(new DropWizardMetricMaker.RestModule());
     modules.add(new LogFileCompressor.Module());
+
+    // Index module shutdown must happen before work queue shutdown, otherwise
+    // work queue can get stuck waiting on index futures that will never return.
+    modules.add(createIndexModule());
+
     modules.add(new WorkQueue.Module());
     modules.add(new ChangeHookRunner.Module());
     modules.add(new ReceiveCommitsExecutorModule());
@@ -351,7 +356,6 @@ public class Daemon extends SiteProgram {
     modules.add(new PluginRestApiModule());
     modules.add(new RestCacheAdminModule());
     modules.add(new GpgModule(config));
-    modules.add(createIndexModule());
     if (MoreObjects.firstNonNull(httpd, true)) {
       modules.add(new CanonicalWebUrlModule() {
         @Override

gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java

@@ -292,7 +292,6 @@ public class WebAppInitializer extends GuiceServletContextListener
   private Injector createSysInjector() {
     final List<Module> modules = new ArrayList<>();
     modules.add(new DropWizardMetricMaker.RestModule());
-    modules.add(new WorkQueue.Module());
     modules.add(new ChangeHookRunner.Module());
     modules.add(new ReceiveCommitsExecutorModule());
     modules.add(new DiffExecutorModule());
@@ -306,13 +305,12 @@ public class WebAppInitializer extends GuiceServletContextListener
     modules.add(new PluginRestApiModule());
     modules.add(new RestCacheAdminModule());
     modules.add(new GpgModule(config));
-    switch (indexType) {
+
-      case LUCENE:
+    // Index module shutdown must happen before work queue shutdown, otherwise
-        modules.add(new LuceneIndexModule());
+    // work queue can get stuck waiting on index futures that will never return.
-        break;
+    modules.add(createIndexModule());
-      default:
+
-        throw new IllegalStateException("unsupported index.type = " + indexType);
+    modules.add(new WorkQueue.Module());
-    }
     modules.add(new CanonicalWebUrlModule() {
       @Override
       protected Class<? extends Provider<String>> provider() {
@@ -332,6 +330,15 @@ public class WebAppInitializer extends GuiceServletContextListener
     return cfgInjector.createChildInjector(modules);
   }
 
+  private Module createIndexModule() {
+    switch (indexType) {
+      case LUCENE:
+        return new LuceneIndexModule();
+      default:
+        throw new IllegalStateException("unsupported index.type = " + indexType);
+    }
+  }
+
   private void initIndexType() {
     indexType = IndexModule.getIndexType(cfgInjector);
   }