Allow graceful rolling restarts

Set a graceful stop timeout for allowing Jetty to wait for incoming
requests to be completed before shutting down its sockets.

This change is mandatory for all of those who are using Gerrit behind
a HAProxy server and want to have rolling restarts that are completely
transparent to the end user.

Change-Id: I5fec738db70df8de963b2ad4701aa394a9209f38

Documentation/config-gerrit.txt

@@ -2408,6 +2408,21 @@ to permit fast restarts.
 +
 By default, true.
 
+[[httpd.gracefulStopTimeout]]httpd.gracefulStopTimeout::
++
+Set a graceful stop time. If set, the daemon ensures that all incoming
+calls are preserved for a maximum period of time, before starting
+the graceful shutdown process. Sites behind a workload balancer such as
+HAProxy would need this to be set for avoiding serving errors during
+rolling restarts.
++
+Values should use common unit suffixes to express their setting:
++
+* s, sec, second, seconds
+* m, min, minute, minutes
++
+By default, 0 seconds (immediate shutdown).
+
 [[httpd.inheritChannel]]httpd.inheritChannel::
 +
 If true, permits the daemon to inherit its server socket channel

gerrit-pgm/src/main/java/com/google/gerrit/pgm/http/jetty/JettyServer.java

@@ -39,6 +39,7 @@ import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import javax.servlet.DispatcherType;
 import javax.servlet.Filter;
 import org.eclipse.jetty.http.HttpScheme;
@@ -56,6 +57,7 @@ import org.eclipse.jetty.server.SslConnectionFactory;
 import org.eclipse.jetty.server.handler.ContextHandler;
 import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.server.handler.RequestLogHandler;
+import org.eclipse.jetty.server.handler.StatisticsHandler;
 import org.eclipse.jetty.server.session.SessionHandler;
 import org.eclipse.jetty.servlet.DefaultServlet;
 import org.eclipse.jetty.servlet.FilterHolder;
@@ -149,6 +151,15 @@ public class JettyServer {
       httpd.addBean(mbean);
     }
 
+    long gracefulStopTimeout =
+        cfg.getTimeUnit("httpd", null, "gracefulStopTimeout", 0L, TimeUnit.MILLISECONDS);
+    if (gracefulStopTimeout > 0) {
+      StatisticsHandler statsHandler = new StatisticsHandler();
+      statsHandler.setHandler(app);
+      app = statsHandler;
+      httpd.setStopTimeout(gracefulStopTimeout);
+    }
+
     httpd.setHandler(app);
     httpd.setStopAtShutdown(false);
   }