Support tracing on clone, fetch and ls-refs

If git protocol V2 is used clients can pass server options on clone,
fetch and ls-refs. Add support for a "trace" server option that enables
request tracing in Gerrit. E.g.:

  git fetch origin -o trace=123
  git ls-remote origin -o trace=456
  git clone --server-option trace=789 <URL>

Note the git clone doesn't support server options by '-o' since git
clone uses '-o' to set the name of the origin.

Tracing git clone, fetch and ls-refs only works if git protocol V2 is
used. This means v2 must be enabled in:

- the repository's .git/config:
  [protocol]
    version = 2
- on the client side, by either passing -c protocol.version=2, or
setting globally in ~/.gitconfig:
  [protocol]
    version = 2

Extend existing GitProtocolV2IT integration test to pass -o trace=<num>
option. As the consequence the trace context is created with this trace
id, e.g.:

DEBUG com.google.gerrit.server.permissions.RefControl : 'user' cannot \
perform 'read' with force=false on project 'foo' for \
ref 'refs/heads/secret' [...] [CONTEXT forced=true TRACE_ID="12345" \
project="foo" ]

However, it is not clear how to verify that the trace context works as
expected.

Another open question is how we could return a generated trace ID to the
client. For now this is left as a todo, because the tracing
functionality is already usable by setting the trace server option with
an explicit trace ID (as shown in the examples above).

Change-Id: I04662a352eb9ceccdbe25eb398289badd07492e2
Signed-off-by: Edwin Kempin <ekempin@google.com>

Documentation/user-request-tracing.txt

@@ -22,12 +22,19 @@ request type:
   `--trace` option. More information about this can be found in
   the link:cmd-index.html#trace[Trace] section of the
   link:cmd-index.html[SSH command documentation].
-* Git: For Git pushes tracing can be enabled by setting the
-  `trace` push option, the trace ID is returned in the command output.
-  More information about this can be found in
-  the link:user-upload.html#trace[Trace] section of the
-  link:user-upload.html[upload documentation]. Tracing for Git requests
-  other than Git push is not supported.
+* Git Push (requires usage of git protocol v2): For Git pushes tracing
+  can be enabled by setting the `trace` push option, the trace ID is
+  returned in the command output. More information about this can be
+  found in the link:user-upload.html#trace[Trace] section of the
+  link:user-upload.html[upload documentation].
+* Git Clone/Fetch/Ls-Remote (requires usage of git protocol v2): For
+  Git clone/fetch/ls-remote tracing can be enabled by setting the
+  `trace` server option. Use '-o trace=<TRACE-ID>' for `git fetch` and
+  `git ls-remote`, and '--server-option trace=<TRACE-ID>' for
+  `git clone`. If the `trace` server option is set without a value
+  (without trace ID) a trace ID is generated but the generated trace ID
+  is not returned to the client (hence a trace ID should always be
+  set).
 
 When request tracing is enabled it is possible to provide an ID that
 should be used as trace ID. If a trace ID is not provided a trace ID is

java/com/google/gerrit/httpd/GitOverHttpServlet.java

@@ -32,6 +32,7 @@ import com.google.gerrit.server.audit.HttpAuditEvent;
 import com.google.gerrit.server.cache.CacheModule;
 import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.git.PermissionAwareRepositoryManager;
+import com.google.gerrit.server.git.TracingHook;
 import com.google.gerrit.server.git.TransferConfig;
 import com.google.gerrit.server.git.UploadPackInitializer;
 import com.google.gerrit.server.git.receive.AsyncReceiveCommits;
@@ -415,7 +416,11 @@ public class GitOverHttpServlet extends GitServlet {
         up.setPreUploadHook(
             PreUploadHookChain.newChain(
                 Lists.newArrayList(up.getPreUploadHook(), uploadValidators)));
-        next.doFilter(httpRequest, responseWrapper);
+
+        try (TracingHook tracingHook = new TracingHook()) {
+          up.setProtocolV2Hook(tracingHook);
+          next.doFilter(httpRequest, responseWrapper);
+        }
       } finally {
         groupAuditService.dispatch(
             new HttpAuditEvent(

java/com/google/gerrit/server/git/TracingHook.java

@@ -0,0 +1,99 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.git;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.gerrit.server.logging.TraceContext;
+import java.util.List;
+import java.util.Optional;
+import org.eclipse.jgit.transport.FetchV2Request;
+import org.eclipse.jgit.transport.LsRefsV2Request;
+import org.eclipse.jgit.transport.ProtocolV2Hook;
+
+/**
+ * Git hook for ls-refs and fetch that enables Gerrit request tracing if the user sets the 'trace'
+ * server option.
+ *
+ * <p>This hook is only invoked if Git protocol v2 is used.
+ *
+ * <p>If the 'trace' server option is specified without value, this means without providing a trace
+ * ID, a trace ID is generated, but it's not returned to the client. Hence users are advised to
+ * always provide a trace ID.
+ */
+public class TracingHook implements ProtocolV2Hook, AutoCloseable {
+  private TraceContext traceContext;
+
+  @Override
+  public void onLsRefs(LsRefsV2Request req) {
+    maybeStartTrace(req.getServerOptions());
+  }
+
+  @Override
+  public void onFetch(FetchV2Request req) {
+    maybeStartTrace(req.getServerOptions());
+  }
+
+  @Override
+  public void close() {
+    if (traceContext != null) {
+      traceContext.close();
+    }
+  }
+
+  /**
+   * Starts request tracing if 'trace' server option is set.
+   *
+   * @param serverOptionList list of provided server options
+   */
+  private void maybeStartTrace(List<String> serverOptionList) {
+    checkState(traceContext == null, "Trace was already started.");
+
+    Optional<String> traceOption = parseTraceOption(serverOptionList);
+    traceContext =
+        TraceContext.newTrace(
+            traceOption.isPresent(),
+            traceOption.orElse(null),
+            (tagName, traceId) -> {
+              // TODO(ekempin): Return trace ID to client
+            });
+  }
+
+  private Optional<String> parseTraceOption(List<String> serverOptionList) {
+    if (serverOptionList == null || serverOptionList.isEmpty()) {
+      return Optional.empty();
+    }
+
+    ListMultimap<String, String> serverOptions = LinkedListMultimap.create();
+    for (String option : serverOptionList) {
+      int e = option.indexOf('=');
+      if (e > 0) {
+        serverOptions.put(option.substring(0, e), option.substring(e + 1));
+      } else {
+        serverOptions.put(option, "");
+      }
+    }
+
+    List<String> traceValues = serverOptions.get("trace");
+    if (!traceValues.isEmpty()) {
+      return Optional.of(Iterables.getLast(traceValues));
+    }
+
+    return Optional.empty();
+  }
+}

java/com/google/gerrit/sshd/commands/Upload.java

@@ -21,6 +21,7 @@ import com.google.gerrit.extensions.restapi.AuthException;
 import com.google.gerrit.server.RequestInfo;
 import com.google.gerrit.server.RequestListener;
 import com.google.gerrit.server.git.PermissionAwareRepositoryManager;
+import com.google.gerrit.server.git.TracingHook;
 import com.google.gerrit.server.git.TransferConfig;
 import com.google.gerrit.server.git.UploadPackInitializer;
 import com.google.gerrit.server.git.validators.UploadValidationException;
@@ -83,13 +84,14 @@ final class Upload extends AbstractGitCommand {
     for (UploadPackInitializer initializer : uploadPackInitializers) {
       initializer.init(projectState.getNameKey(), up);
     }
-    try (TraceContext traceContext = TraceContext.open()) {
+    try (TraceContext traceContext = TraceContext.open();
+        TracingHook tracingHook = new TracingHook()) {
       RequestInfo requestInfo =
           RequestInfo.builder(RequestInfo.RequestType.GIT_UPLOAD, user, traceContext)
               .project(projectState.getNameKey())
               .build();
       requestListeners.runEach(l -> l.onRequest(requestInfo));
-
+      up.setProtocolV2Hook(tracingHook);
       up.upload(in, out, err);
       session.setPeerAgent(up.getPeerUserAgent());
     } catch (UploadValidationException e) {

javatests/com/google/gerrit/integration/git/GitProtocolV2IT.java

@@ -47,7 +47,7 @@ public class GitProtocolV2IT extends StandaloneSiteTest {
   private final String[] SSH_KEYGEN_CMD =
       new String[] {"ssh-keygen", "-t", "rsa", "-q", "-P", "", "-f"};
   private final String[] GIT_LS_REMOTE =
-      new String[] {"git", "-c", "protocol.version=2", "ls-remote"};
+      new String[] {"git", "-c", "protocol.version=2", "ls-remote", "-o", "trace=12345"};
   private final String GIT_SSH_COMMAND =
       "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i";