Support tracing for single REST requests

To investigate why a certain REST call failed one needs to find matching
messages in the logs. Sometimes this is not as easy as it sounds:

1) Error reports may not tell exactly which REST endpoint was invoked
   and what were the IDs of the REST resources on which it was invoked.
2) For busy servers finding logs that correspond to the failing REST
   call may be difficult because of the huge amount of logs. Matching
   logs by the timestamp of when the request was done may also not be
   straight-forward if the user is in a different timezone than the
   server.
3) The error log normally only contains errors, but no detailed debug
   information.
4) Enabling debug logs effects all requests and usually results in a lot
   of log spam.

The idea is to enable debug traces on need for single requests:

a) A debug trace is enabled by setting the 'trace' request parameter
b) The response contains an 'X-Gerrit-Trace' header with the trace ID
c) An administrator can use the trace ID to find all logs for this
   request in the error log.
d) The debug trace can contain additional debug information which is not
   written when tracing is disabled.

This change implements a), b) and c) for REST requests. If the trace
parameter is set for a REST request we open a trace context that sets a
logging tag with a generated trace ID that is then automatically
included into all log statements that are triggered by this request. The
trace ID is returned to the client as 'X-Gerrit-Trace' header in the
response.

d) will be implemented by a follow-up change which will make use of
Flogger's force log functionality.

Change-Id: Ic79aadedbe5d73e94daefa6aa9d207edbffc1607
Signed-off-by: Edwin Kempin <ekempin@google.com>

java/com/google/gerrit/httpd/restapi/ParameterParser.java

@@ -56,8 +56,10 @@ import javax.servlet.http.HttpServletResponse;
 import org.kohsuke.args4j.CmdLineException;
 
 public class ParameterParser {
+  public static final String TRACE_PARAMETER = "trace";
+
   private static final ImmutableSet<String> RESERVED_KEYS =
-      ImmutableSet.of("pp", "prettyPrint", "strict", "callback", "alt", "fields");
+      ImmutableSet.of("pp", "prettyPrint", "strict", "callback", "alt", "fields", TRACE_PARAMETER);
 
   @AutoValue
   public abstract static class QueryParams {

java/com/google/gerrit/httpd/restapi/RestApiServlet.java

@@ -48,6 +48,7 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static javax.servlet.http.HttpServletResponse.SC_PRECONDITION_FAILED;
 import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.CharMatcher;
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
@@ -107,10 +108,12 @@ import com.google.gerrit.server.audit.ExtendedHttpAuditEvent;
 import com.google.gerrit.server.cache.PerThreadCache;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.git.LockFailureException;
+import com.google.gerrit.server.logging.TraceContext;
 import com.google.gerrit.server.permissions.GlobalPermission;
 import com.google.gerrit.server.permissions.PermissionBackend;
 import com.google.gerrit.server.permissions.PermissionBackendException;
 import com.google.gerrit.server.update.UpdateException;
+import com.google.gerrit.server.util.RequestId;
 import com.google.gerrit.util.http.CacheHeaders;
 import com.google.gerrit.util.http.RequestUtil;
 import com.google.gson.ExclusionStrategy;
@@ -177,6 +180,8 @@ public class RestApiServlet extends HttpServlet {
 
   private static final String FORM_TYPE = "application/x-www-form-urlencoded";
 
+  @VisibleForTesting public static final String X_GERRIT_TRACE = "X-Gerrit-Trace";
+
   // HTTP 422 Unprocessable Entity.
   // TODO: Remove when HttpServletResponse.SC_UNPROCESSABLE_ENTITY is available
   private static final int SC_UNPROCESSABLE_ENTITY = 422;
@@ -280,332 +285,337 @@ public class RestApiServlet extends HttpServlet {
     RestResource rsrc = TopLevelResource.INSTANCE;
     ViewData viewData = null;
 
-    try (PerThreadCache ignored = PerThreadCache.create()) {
-      if (isCorsPreflight(req)) {
-        doCorsPreflight(req, res);
-        return;
-      }
+    try (TraceContext traceContext = enableTracing(req, res)) {
+      try (PerThreadCache ignored = PerThreadCache.create()) {
 
-      qp = ParameterParser.getQueryParams(req);
-      checkCors(req, res, qp.hasXdOverride());
-      if (qp.hasXdOverride()) {
-        req = applyXdOverrides(req, qp);
-      }
-      checkUserSession(req);
-
-      List<IdString> path = splitPath(req);
-      RestCollection<RestResource, RestResource> rc = members.get();
-      globals
-          .permissionBackend
-          .currentUser()
-          .checkAny(GlobalPermission.fromAnnotation(rc.getClass()));
-
-      viewData = new ViewData(null, null);
-
-      if (path.isEmpty()) {
-        if (rc instanceof NeedsParams) {
-          ((NeedsParams) rc).setParams(qp.params());
+        if (isCorsPreflight(req)) {
+          doCorsPreflight(req, res);
+          return;
         }
 
-        if (isRead(req)) {
-          viewData = new ViewData(null, rc.list());
-        } else if (isPost(req)) {
-          RestView<RestResource> restCollectionView =
-              rc.views().get("gerrit", "POST_ON_COLLECTION./");
-          if (restCollectionView != null) {
-            viewData = new ViewData(null, restCollectionView);
-          } else {
-            throw methodNotAllowed(req);
-          }
-        } else {
-          // DELETE on root collections is not supported
-          throw methodNotAllowed(req);
+        qp = ParameterParser.getQueryParams(req);
+        checkCors(req, res, qp.hasXdOverride());
+        if (qp.hasXdOverride()) {
+          req = applyXdOverrides(req, qp);
         }
-      } else {
-        IdString id = path.remove(0);
-        try {
-          rsrc = rc.parse(rsrc, id);
-          if (path.isEmpty()) {
-            checkPreconditions(req);
-          }
-        } catch (ResourceNotFoundException e) {
-          if (!path.isEmpty()) {
-            throw e;
-          }
+        checkUserSession(req);
 
-          if (isPost(req) || isPut(req)) {
-            RestView<RestResource> createView = rc.views().get("gerrit", "CREATE./");
-            if (createView != null) {
-              viewData = new ViewData(null, createView);
-              status = SC_CREATED;
-              path.add(id);
-            } else {
-              throw e;
-            }
-          } else if (isDelete(req)) {
-            RestView<RestResource> deleteView = rc.views().get("gerrit", "DELETE_MISSING./");
-            if (deleteView != null) {
-              viewData = new ViewData(null, deleteView);
-              status = SC_NO_CONTENT;
-              path.add(id);
-            } else {
-              throw e;
-            }
-          } else {
-            throw e;
-          }
-        }
-        if (viewData.view == null) {
-          viewData = view(rc, req.getMethod(), path);
-        }
-      }
-      checkRequiresCapability(viewData);
+        List<IdString> path = splitPath(req);
+        RestCollection<RestResource, RestResource> rc = members.get();
+        globals
+            .permissionBackend
+            .currentUser()
+            .checkAny(GlobalPermission.fromAnnotation(rc.getClass()));
 
-      while (viewData.view instanceof RestCollection<?, ?>) {
-        @SuppressWarnings("unchecked")
-        RestCollection<RestResource, RestResource> c =
-            (RestCollection<RestResource, RestResource>) viewData.view;
+        viewData = new ViewData(null, null);
 
         if (path.isEmpty()) {
+          if (rc instanceof NeedsParams) {
+            ((NeedsParams) rc).setParams(qp.params());
+          }
+
           if (isRead(req)) {
-            viewData = new ViewData(null, c.list());
+            viewData = new ViewData(null, rc.list());
           } else if (isPost(req)) {
             RestView<RestResource> restCollectionView =
-                c.views().get(viewData.pluginName, "POST_ON_COLLECTION./");
-            if (restCollectionView != null) {
-              viewData = new ViewData(null, restCollectionView);
-            } else {
-              throw methodNotAllowed(req);
-            }
-          } else if (isDelete(req)) {
-            RestView<RestResource> restCollectionView =
-                c.views().get(viewData.pluginName, "DELETE_ON_COLLECTION./");
+                rc.views().get("gerrit", "POST_ON_COLLECTION./");
             if (restCollectionView != null) {
               viewData = new ViewData(null, restCollectionView);
             } else {
               throw methodNotAllowed(req);
             }
           } else {
+            // DELETE on root collections is not supported
             throw methodNotAllowed(req);
           }
-          break;
-        }
-        IdString id = path.remove(0);
-        try {
-          rsrc = c.parse(rsrc, id);
-          checkPreconditions(req);
-          viewData = new ViewData(null, null);
-        } catch (ResourceNotFoundException e) {
-          if (!path.isEmpty()) {
-            throw e;
-          }
+        } else {
+          IdString id = path.remove(0);
+          try {
+            rsrc = rc.parse(rsrc, id);
+            if (path.isEmpty()) {
+              checkPreconditions(req);
+            }
+          } catch (ResourceNotFoundException e) {
+            if (!path.isEmpty()) {
+              throw e;
+            }
 
-          if (isPost(req) || isPut(req)) {
-            RestView<RestResource> createView = c.views().get("gerrit", "CREATE./");
-            if (createView != null) {
-              viewData = new ViewData(null, createView);
-              status = SC_CREATED;
-              path.add(id);
+            if (isPost(req) || isPut(req)) {
+              RestView<RestResource> createView = rc.views().get("gerrit", "CREATE./");
+              if (createView != null) {
+                viewData = new ViewData(null, createView);
+                status = SC_CREATED;
+                path.add(id);
+              } else {
+                throw e;
+              }
+            } else if (isDelete(req)) {
+              RestView<RestResource> deleteView = rc.views().get("gerrit", "DELETE_MISSING./");
+              if (deleteView != null) {
+                viewData = new ViewData(null, deleteView);
+                status = SC_NO_CONTENT;
+                path.add(id);
+              } else {
+                throw e;
+              }
             } else {
               throw e;
             }
-          } else if (isDelete(req)) {
-            RestView<RestResource> deleteView = c.views().get("gerrit", "DELETE_MISSING./");
-            if (deleteView != null) {
-              viewData = new ViewData(null, deleteView);
-              status = SC_NO_CONTENT;
-              path.add(id);
-            } else {
-              throw e;
-            }
-          } else {
-            throw e;
           }
-        }
-        if (viewData.view == null) {
-          viewData = view(c, req.getMethod(), path);
+          if (viewData.view == null) {
+            viewData = view(rc, req.getMethod(), path);
+          }
         }
         checkRequiresCapability(viewData);
-      }
 
-      if (notModified(req, rsrc, viewData.view)) {
-        res.sendError(SC_NOT_MODIFIED);
-        return;
-      }
+        while (viewData.view instanceof RestCollection<?, ?>) {
+          @SuppressWarnings("unchecked")
+          RestCollection<RestResource, RestResource> c =
+              (RestCollection<RestResource, RestResource>) viewData.view;
 
-      if (!globals.paramParser.get().parse(viewData.view, qp.params(), req, res)) {
-        return;
-      }
-
-      if (viewData.view instanceof RestReadView<?> && isRead(req)) {
-        result = ((RestReadView<RestResource>) viewData.view).apply(rsrc);
-      } else if (viewData.view instanceof RestModifyView<?, ?>) {
-        @SuppressWarnings("unchecked")
-        RestModifyView<RestResource, Object> m =
-            (RestModifyView<RestResource, Object>) viewData.view;
-
-        Type type = inputType(m);
-        inputRequestBody = parseRequest(req, type);
-        result = m.apply(rsrc, inputRequestBody);
-        if (inputRequestBody instanceof RawInput) {
-          try (InputStream is = req.getInputStream()) {
-            ServletUtils.consumeRequestBody(is);
+          if (path.isEmpty()) {
+            if (isRead(req)) {
+              viewData = new ViewData(null, c.list());
+            } else if (isPost(req)) {
+              RestView<RestResource> restCollectionView =
+                  c.views().get(viewData.pluginName, "POST_ON_COLLECTION./");
+              if (restCollectionView != null) {
+                viewData = new ViewData(null, restCollectionView);
+              } else {
+                throw methodNotAllowed(req);
+              }
+            } else if (isDelete(req)) {
+              RestView<RestResource> restCollectionView =
+                  c.views().get(viewData.pluginName, "DELETE_ON_COLLECTION./");
+              if (restCollectionView != null) {
+                viewData = new ViewData(null, restCollectionView);
+              } else {
+                throw methodNotAllowed(req);
+              }
+            } else {
+              throw methodNotAllowed(req);
+            }
+            break;
           }
-        }
-      } else if (viewData.view instanceof RestCollectionCreateView<?, ?, ?>) {
-        @SuppressWarnings("unchecked")
-        RestCollectionCreateView<RestResource, RestResource, Object> m =
-            (RestCollectionCreateView<RestResource, RestResource, Object>) viewData.view;
+          IdString id = path.remove(0);
+          try {
+            rsrc = c.parse(rsrc, id);
+            checkPreconditions(req);
+            viewData = new ViewData(null, null);
+          } catch (ResourceNotFoundException e) {
+            if (!path.isEmpty()) {
+              throw e;
+            }
 
-        Type type = inputType(m);
-        inputRequestBody = parseRequest(req, type);
-        result = m.apply(rsrc, path.get(0), inputRequestBody);
-        if (inputRequestBody instanceof RawInput) {
-          try (InputStream is = req.getInputStream()) {
-            ServletUtils.consumeRequestBody(is);
+            if (isPost(req) || isPut(req)) {
+              RestView<RestResource> createView = c.views().get("gerrit", "CREATE./");
+              if (createView != null) {
+                viewData = new ViewData(null, createView);
+                status = SC_CREATED;
+                path.add(id);
+              } else {
+                throw e;
+              }
+            } else if (isDelete(req)) {
+              RestView<RestResource> deleteView = c.views().get("gerrit", "DELETE_MISSING./");
+              if (deleteView != null) {
+                viewData = new ViewData(null, deleteView);
+                status = SC_NO_CONTENT;
+                path.add(id);
+              } else {
+                throw e;
+              }
+            } else {
+              throw e;
+            }
           }
-        }
-      } else if (viewData.view instanceof RestCollectionDeleteMissingView<?, ?, ?>) {
-        @SuppressWarnings("unchecked")
-        RestCollectionDeleteMissingView<RestResource, RestResource, Object> m =
-            (RestCollectionDeleteMissingView<RestResource, RestResource, Object>) viewData.view;
-
-        Type type = inputType(m);
-        inputRequestBody = parseRequest(req, type);
-        result = m.apply(rsrc, path.get(0), inputRequestBody);
-        if (inputRequestBody instanceof RawInput) {
-          try (InputStream is = req.getInputStream()) {
-            ServletUtils.consumeRequestBody(is);
+          if (viewData.view == null) {
+            viewData = view(c, req.getMethod(), path);
           }
+          checkRequiresCapability(viewData);
         }
-      } else if (viewData.view instanceof RestCollectionModifyView<?, ?, ?>) {
-        @SuppressWarnings("unchecked")
-        RestCollectionModifyView<RestResource, RestResource, Object> m =
-            (RestCollectionModifyView<RestResource, RestResource, Object>) viewData.view;
 
-        Type type = inputType(m);
-        inputRequestBody = parseRequest(req, type);
-        result = m.apply(rsrc, inputRequestBody);
-        if (inputRequestBody instanceof RawInput) {
-          try (InputStream is = req.getInputStream()) {
-            ServletUtils.consumeRequestBody(is);
+        if (notModified(req, rsrc, viewData.view)) {
+          res.sendError(SC_NOT_MODIFIED);
+          return;
+        }
+
+        if (!globals.paramParser.get().parse(viewData.view, qp.params(), req, res)) {
+          return;
+        }
+
+        if (viewData.view instanceof RestReadView<?> && isRead(req)) {
+          result = ((RestReadView<RestResource>) viewData.view).apply(rsrc);
+        } else if (viewData.view instanceof RestModifyView<?, ?>) {
+          @SuppressWarnings("unchecked")
+          RestModifyView<RestResource, Object> m =
+              (RestModifyView<RestResource, Object>) viewData.view;
+
+          Type type = inputType(m);
+          inputRequestBody = parseRequest(req, type);
+          result = m.apply(rsrc, inputRequestBody);
+          if (inputRequestBody instanceof RawInput) {
+            try (InputStream is = req.getInputStream()) {
+              ServletUtils.consumeRequestBody(is);
+            }
           }
-        }
-      } else {
-        throw new ResourceNotFoundException();
-      }
+        } else if (viewData.view instanceof RestCollectionCreateView<?, ?, ?>) {
+          @SuppressWarnings("unchecked")
+          RestCollectionCreateView<RestResource, RestResource, Object> m =
+              (RestCollectionCreateView<RestResource, RestResource, Object>) viewData.view;
 
-      if (result instanceof Response) {
-        @SuppressWarnings("rawtypes")
-        Response<?> r = (Response) result;
-        status = r.statusCode();
-        configureCaching(req, res, rsrc, viewData.view, r.caching());
-      } else if (result instanceof Response.Redirect) {
-        CacheHeaders.setNotCacheable(res);
-        res.sendRedirect(((Response.Redirect) result).location());
-        return;
-      } else if (result instanceof Response.Accepted) {
-        CacheHeaders.setNotCacheable(res);
-        res.setStatus(SC_ACCEPTED);
-        res.setHeader(HttpHeaders.LOCATION, ((Response.Accepted) result).location());
-        return;
-      } else {
-        CacheHeaders.setNotCacheable(res);
-      }
-      res.setStatus(status);
+          Type type = inputType(m);
+          inputRequestBody = parseRequest(req, type);
+          result = m.apply(rsrc, path.get(0), inputRequestBody);
+          if (inputRequestBody instanceof RawInput) {
+            try (InputStream is = req.getInputStream()) {
+              ServletUtils.consumeRequestBody(is);
+            }
+          }
+        } else if (viewData.view instanceof RestCollectionDeleteMissingView<?, ?, ?>) {
+          @SuppressWarnings("unchecked")
+          RestCollectionDeleteMissingView<RestResource, RestResource, Object> m =
+              (RestCollectionDeleteMissingView<RestResource, RestResource, Object>) viewData.view;
 
-      if (result != Response.none()) {
-        result = Response.unwrap(result);
-        if (result instanceof BinaryResult) {
-          responseBytes = replyBinaryResult(req, res, (BinaryResult) result);
+          Type type = inputType(m);
+          inputRequestBody = parseRequest(req, type);
+          result = m.apply(rsrc, path.get(0), inputRequestBody);
+          if (inputRequestBody instanceof RawInput) {
+            try (InputStream is = req.getInputStream()) {
+              ServletUtils.consumeRequestBody(is);
+            }
+          }
+        } else if (viewData.view instanceof RestCollectionModifyView<?, ?, ?>) {
+          @SuppressWarnings("unchecked")
+          RestCollectionModifyView<RestResource, RestResource, Object> m =
+              (RestCollectionModifyView<RestResource, RestResource, Object>) viewData.view;
+
+          Type type = inputType(m);
+          inputRequestBody = parseRequest(req, type);
+          result = m.apply(rsrc, inputRequestBody);
+          if (inputRequestBody instanceof RawInput) {
+            try (InputStream is = req.getInputStream()) {
+              ServletUtils.consumeRequestBody(is);
+            }
+          }
         } else {
-          responseBytes = replyJson(req, res, qp.config(), result);
+          throw new ResourceNotFoundException();
         }
-      }
-    } catch (MalformedJsonException | JsonParseException e) {
-      responseBytes =
-          replyError(req, res, status = SC_BAD_REQUEST, "Invalid " + JSON_TYPE + " in request", e);
-    } catch (BadRequestException e) {
-      responseBytes =
-          replyError(
-              req, res, status = SC_BAD_REQUEST, messageOr(e, "Bad Request"), e.caching(), e);
-    } catch (AuthException e) {
-      responseBytes =
-          replyError(req, res, status = SC_FORBIDDEN, messageOr(e, "Forbidden"), e.caching(), e);
-    } catch (AmbiguousViewException e) {
-      responseBytes = replyError(req, res, status = SC_NOT_FOUND, messageOr(e, "Ambiguous"), e);
-    } catch (ResourceNotFoundException e) {
-      responseBytes =
-          replyError(req, res, status = SC_NOT_FOUND, messageOr(e, "Not Found"), e.caching(), e);
-    } catch (MethodNotAllowedException e) {
-      responseBytes =
-          replyError(
-              req,
-              res,
-              status = SC_METHOD_NOT_ALLOWED,
-              messageOr(e, "Method Not Allowed"),
-              e.caching(),
-              e);
-    } catch (ResourceConflictException e) {
-      responseBytes =
-          replyError(req, res, status = SC_CONFLICT, messageOr(e, "Conflict"), e.caching(), e);
-    } catch (PreconditionFailedException e) {
-      responseBytes =
-          replyError(
-              req,
-              res,
-              status = SC_PRECONDITION_FAILED,
-              messageOr(e, "Precondition Failed"),
-              e.caching(),
-              e);
-    } catch (UnprocessableEntityException e) {
-      responseBytes =
-          replyError(
-              req,
-              res,
-              status = SC_UNPROCESSABLE_ENTITY,
-              messageOr(e, "Unprocessable Entity"),
-              e.caching(),
-              e);
-    } catch (NotImplementedException e) {
-      responseBytes =
-          replyError(req, res, status = SC_NOT_IMPLEMENTED, messageOr(e, "Not Implemented"), e);
-    } catch (UpdateException e) {
-      Throwable t = e.getCause();
-      if (t instanceof LockFailureException) {
+
+        if (result instanceof Response) {
+          @SuppressWarnings("rawtypes")
+          Response<?> r = (Response) result;
+          status = r.statusCode();
+          configureCaching(req, res, rsrc, viewData.view, r.caching());
+        } else if (result instanceof Response.Redirect) {
+          CacheHeaders.setNotCacheable(res);
+          res.sendRedirect(((Response.Redirect) result).location());
+          return;
+        } else if (result instanceof Response.Accepted) {
+          CacheHeaders.setNotCacheable(res);
+          res.setStatus(SC_ACCEPTED);
+          res.setHeader(HttpHeaders.LOCATION, ((Response.Accepted) result).location());
+          return;
+        } else {
+          CacheHeaders.setNotCacheable(res);
+        }
+        res.setStatus(status);
+
+        if (result != Response.none()) {
+          result = Response.unwrap(result);
+          if (result instanceof BinaryResult) {
+            responseBytes = replyBinaryResult(req, res, (BinaryResult) result);
+          } else {
+            responseBytes = replyJson(req, res, qp.config(), result);
+          }
+        }
+      } catch (MalformedJsonException | JsonParseException e) {
         responseBytes =
-            replyError(req, res, status = SC_SERVICE_UNAVAILABLE, messageOr(t, "Lock failure"), e);
-      } else {
+            replyError(
+                req, res, status = SC_BAD_REQUEST, "Invalid " + JSON_TYPE + " in request", e);
+      } catch (BadRequestException e) {
+        responseBytes =
+            replyError(
+                req, res, status = SC_BAD_REQUEST, messageOr(e, "Bad Request"), e.caching(), e);
+      } catch (AuthException e) {
+        responseBytes =
+            replyError(req, res, status = SC_FORBIDDEN, messageOr(e, "Forbidden"), e.caching(), e);
+      } catch (AmbiguousViewException e) {
+        responseBytes = replyError(req, res, status = SC_NOT_FOUND, messageOr(e, "Ambiguous"), e);
+      } catch (ResourceNotFoundException e) {
+        responseBytes =
+            replyError(req, res, status = SC_NOT_FOUND, messageOr(e, "Not Found"), e.caching(), e);
+      } catch (MethodNotAllowedException e) {
+        responseBytes =
+            replyError(
+                req,
+                res,
+                status = SC_METHOD_NOT_ALLOWED,
+                messageOr(e, "Method Not Allowed"),
+                e.caching(),
+                e);
+      } catch (ResourceConflictException e) {
+        responseBytes =
+            replyError(req, res, status = SC_CONFLICT, messageOr(e, "Conflict"), e.caching(), e);
+      } catch (PreconditionFailedException e) {
+        responseBytes =
+            replyError(
+                req,
+                res,
+                status = SC_PRECONDITION_FAILED,
+                messageOr(e, "Precondition Failed"),
+                e.caching(),
+                e);
+      } catch (UnprocessableEntityException e) {
+        responseBytes =
+            replyError(
+                req,
+                res,
+                status = SC_UNPROCESSABLE_ENTITY,
+                messageOr(e, "Unprocessable Entity"),
+                e.caching(),
+                e);
+      } catch (NotImplementedException e) {
+        responseBytes =
+            replyError(req, res, status = SC_NOT_IMPLEMENTED, messageOr(e, "Not Implemented"), e);
+      } catch (UpdateException e) {
+        Throwable t = e.getCause();
+        if (t instanceof LockFailureException) {
+          responseBytes =
+              replyError(
+                  req, res, status = SC_SERVICE_UNAVAILABLE, messageOr(t, "Lock failure"), e);
+        } else {
+          status = SC_INTERNAL_SERVER_ERROR;
+          responseBytes = handleException(e, req, res);
+        }
+      } catch (Exception e) {
         status = SC_INTERNAL_SERVER_ERROR;
         responseBytes = handleException(e, req, res);
+      } finally {
+        String metric =
+            viewData != null && viewData.view != null ? globals.metrics.view(viewData) : "_unknown";
+        globals.metrics.count.increment(metric);
+        if (status >= SC_BAD_REQUEST) {
+          globals.metrics.errorCount.increment(metric, status);
+        }
+        if (responseBytes != -1) {
+          globals.metrics.responseBytes.record(metric, responseBytes);
+        }
+        globals.metrics.serverLatency.record(
+            metric, System.nanoTime() - startNanos, TimeUnit.NANOSECONDS);
+        globals.auditService.dispatch(
+            new ExtendedHttpAuditEvent(
+                globals.webSession.get().getSessionId(),
+                globals.currentUser.get(),
+                req,
+                auditStartTs,
+                qp != null ? qp.params() : ImmutableListMultimap.of(),
+                inputRequestBody,
+                status,
+                result,
+                rsrc,
+                viewData == null ? null : viewData.view));
       }
-    } catch (Exception e) {
-      status = SC_INTERNAL_SERVER_ERROR;
-      responseBytes = handleException(e, req, res);
-    } finally {
-      String metric =
-          viewData != null && viewData.view != null ? globals.metrics.view(viewData) : "_unknown";
-      globals.metrics.count.increment(metric);
-      if (status >= SC_BAD_REQUEST) {
-        globals.metrics.errorCount.increment(metric, status);
-      }
-      if (responseBytes != -1) {
-        globals.metrics.responseBytes.record(metric, responseBytes);
-      }
-      globals.metrics.serverLatency.record(
-          metric, System.nanoTime() - startNanos, TimeUnit.NANOSECONDS);
-      globals.auditService.dispatch(
-          new ExtendedHttpAuditEvent(
-              globals.webSession.get().getSessionId(),
-              globals.currentUser.get(),
-              req,
-              auditStartTs,
-              qp != null ? qp.params() : ImmutableListMultimap.of(),
-              inputRequestBody,
-              status,
-              result,
-              rsrc,
-              viewData == null ? null : viewData.view));
     }
   }
 
@@ -1265,6 +1275,16 @@ public class RestApiServlet extends HttpServlet {
     }
   }
 
+  private TraceContext enableTracing(HttpServletRequest req, HttpServletResponse res) {
+    String v = req.getParameter(ParameterParser.TRACE_PARAMETER);
+    if (v != null && (v.isEmpty() || Boolean.parseBoolean(v))) {
+      RequestId traceId = new RequestId();
+      res.setHeader(X_GERRIT_TRACE, traceId.toString());
+      return new TraceContext(RequestId.Type.TRACE_ID, traceId);
+    }
+    return TraceContext.DISABLED;
+  }
+
   private boolean isDelete(HttpServletRequest req) {
     return "DELETE".equals(req.getMethod());
   }

java/com/google/gerrit/server/logging/TraceContext.java

@@ -19,6 +19,8 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import com.google.gerrit.server.util.RequestId;
 
 public class TraceContext implements AutoCloseable {
+  public static final TraceContext DISABLED = new TraceContext();
+
   private final String tagName;
   private final String tagValue;
   private final boolean removeOnClose;
@@ -33,6 +35,12 @@ public class TraceContext implements AutoCloseable {
     this.removeOnClose = LoggingContext.getInstance().addTag(this.tagName, this.tagValue);
   }
 
+  private TraceContext() {
+    this.tagName = null;
+    this.tagValue = null;
+    this.removeOnClose = false;
+  }
+
   @Override
   public void close() {
     if (removeOnClose) {

java/com/google/gerrit/server/util/RequestId.java

@@ -17,6 +17,7 @@ package com.google.gerrit.server.util;
 import com.google.common.base.Enums;
 import com.google.common.hash.Hasher;
 import com.google.common.hash.Hashing;
+import com.google.gerrit.common.Nullable;
 import com.google.gerrit.common.TimeUtil;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.reviewdb.client.Project;
@@ -40,7 +41,8 @@ public class RequestId {
 
   public enum Type {
     RECEIVE_ID,
-    SUBMISSION_ID;
+    SUBMISSION_ID,
+    TRACE_ID;
 
     static boolean isId(String id) {
       return id != null && Enums.getIfPresent(Type.class, id).isPresent();
@@ -61,10 +63,18 @@ public class RequestId {
 
   private final String str;
 
-  private RequestId(String resourceId) {
+  public RequestId() {
+    this(null);
+  }
+
+  private RequestId(@Nullable String resourceId) {
     Hasher h = Hashing.murmur3_128().newHasher();
     h.putLong(Thread.currentThread().getId()).putUnencodedChars(MACHINE_ID);
-    str = resourceId + "-" + TimeUtil.nowTs().getTime() + "-" + h.hash().toString().substring(0, 8);
+    str =
+        (resourceId != null ? resourceId + "-" : "")
+            + TimeUtil.nowTs().getTime()
+            + "-"
+            + h.hash().toString().substring(0, 8);
   }
 
   @Override

javatests/com/google/gerrit/acceptance/rest/TraceIT.java

@@ -0,0 +1,96 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.acceptance.rest;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.apache.http.HttpStatus.SC_CREATED;
+
+import com.google.gerrit.acceptance.AbstractDaemonTest;
+import com.google.gerrit.acceptance.RestResponse;
+import com.google.gerrit.extensions.registration.DynamicSet;
+import com.google.gerrit.extensions.registration.RegistrationHandle;
+import com.google.gerrit.httpd.restapi.ParameterParser;
+import com.google.gerrit.httpd.restapi.RestApiServlet;
+import com.google.gerrit.server.logging.LoggingContext;
+import com.google.gerrit.server.project.CreateProjectArgs;
+import com.google.gerrit.server.validators.ProjectCreationValidationListener;
+import com.google.gerrit.server.validators.ValidationException;
+import com.google.inject.Inject;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TraceIT extends AbstractDaemonTest {
+  @Inject private DynamicSet<ProjectCreationValidationListener> projectCreationValidationListeners;
+
+  private TraceValidatingProjectCreationValidationListener listener;
+  private RegistrationHandle registrationHandle;
+
+  @Before
+  public void setup() {
+    listener = new TraceValidatingProjectCreationValidationListener();
+    registrationHandle = projectCreationValidationListeners.add(listener);
+  }
+
+  @After
+  public void cleanup() {
+    registrationHandle.remove();
+  }
+
+  @Test
+  public void withoutTrace() throws Exception {
+    RestResponse response = adminRestSession.put("/projects/new1");
+    assertThat(response.getStatusCode()).isEqualTo(SC_CREATED);
+    assertThat(response.getHeader(RestApiServlet.X_GERRIT_TRACE)).isNull();
+    assertThat(listener.foundTraceId).isFalse();
+  }
+
+  @Test
+  public void withTrace() throws Exception {
+    RestResponse response =
+        adminRestSession.put("/projects/new2?" + ParameterParser.TRACE_PARAMETER);
+    assertThat(response.getStatusCode()).isEqualTo(SC_CREATED);
+    assertThat(response.getHeader(RestApiServlet.X_GERRIT_TRACE)).isNotNull();
+    assertThat(listener.foundTraceId).isTrue();
+  }
+
+  @Test
+  public void withTraceTrue() throws Exception {
+    RestResponse response =
+        adminRestSession.put("/projects/new3?" + ParameterParser.TRACE_PARAMETER + "=true");
+    assertThat(response.getStatusCode()).isEqualTo(SC_CREATED);
+    assertThat(response.getHeader(RestApiServlet.X_GERRIT_TRACE)).isNotNull();
+    assertThat(listener.foundTraceId).isTrue();
+  }
+
+  @Test
+  public void withTraceFalse() throws Exception {
+    RestResponse response =
+        adminRestSession.put("/projects/new4?" + ParameterParser.TRACE_PARAMETER + "=false");
+    assertThat(response.getStatusCode()).isEqualTo(SC_CREATED);
+    assertThat(response.getHeader(RestApiServlet.X_GERRIT_TRACE)).isNull();
+    assertThat(listener.foundTraceId).isFalse();
+  }
+
+  private static class TraceValidatingProjectCreationValidationListener
+      implements ProjectCreationValidationListener {
+    Boolean foundTraceId;
+
+    @Override
+    public void validateNewProject(CreateProjectArgs args) throws ValidationException {
+      this.foundTraceId = LoggingContext.getInstance().getTagsAsMap().containsKey("TRACE_ID");
+    }
+  }
+}