Support 'git-upload-archive'

This allows use the standard git archive command to create an archive
of the content of a repository:

  $ git archive -f tar.bz2 --prefix=foo-1.0/ \
    --remote=ssh://john@gerrit:29418/foo \
    refs/changes/73/673/1 > foo-1.0.tar.bz2

Different compression levels can be configured for zip format:

  $ git archive -f zip -9 \
    --remote=ssh://john@gerrit:29418/foo \
    refs/changes/73/673/1 > foo.zip

TEST PLAN:

  buck test --include ssh

Bug: Issue 2061
Change-Id: Ifc1a92bacef3155cf474adee883cbe587dd8759f

Documentation/config-gerrit.txt

@@ -1451,7 +1451,7 @@ downloads are allowed.
 [[download.archive]]download.archive::
 +
 Specifies which archive formats, if any, should be offered on the change
-screen:
+screen and supported for `git-upload-archive` operation:
 +
 ----
 [download]
@@ -1459,11 +1459,17 @@ screen:
   archive = tbz2
   archive = tgz
   archive = txz
+  archive = zip
 ----
 
 If `download.archive` is not specified defaults to all archive
 commands. Set to `off` or empty string to disable.
 
+Zip is not supported because it may be interpreted by a Java plugin as a
+valid JAR file, whose code would have access to cookies on the domain.
+For this reason `zip` format is always excluded from formats offered
+through the `Download` drop down or accessible in the REST API.
+
 [[gc]]
 === Section gc
 

gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/ConfigAnnotationParser.java

@@ -15,11 +15,13 @@
 package com.google.gerrit.acceptance;
 
 import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 
 import org.eclipse.jgit.lib.Config;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 
 class ConfigAnnotationParser {
   private static Splitter splitter = Splitter.on(".").trimResults();
@@ -45,9 +47,19 @@ class ConfigAnnotationParser {
   private static void parseAnnotation(Config cfg, GerritConfig c) {
     ArrayList<String> l = Lists.newArrayList(splitter.split(c.name()));
     if (l.size() == 2) {
-      cfg.setString(l.get(0), null, l.get(1), c.value());
+      if (!Strings.isNullOrEmpty(c.value())) {
+        cfg.setString(l.get(0), null, l.get(1), c.value());
+      } else {
+        String[] values = c.values();
+        cfg.setStringList(l.get(0), null, l.get(1), Arrays.asList(values));
+      }
     } else if (l.size() == 3) {
-      cfg.setString(l.get(0), l.get(1), l.get(2), c.value());
+      if (!Strings.isNullOrEmpty(c.value())) {
+        cfg.setString(l.get(0), l.get(1), l.get(2), c.value());
+      } else {
+        cfg.setStringList(l.get(0), l.get(1), l.get(2),
+            Arrays.asList(c.value()));
+      }
     } else {
       throw new IllegalArgumentException(
           "GerritConfig.name must be of the format"

gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/GerritConfig.java

@@ -24,5 +24,6 @@ import java.lang.annotation.Target;
 @Retention(RUNTIME)
 public @interface GerritConfig {
   String name();
-  String value();
+  String value() default "";
+  String[] values() default "";
 }

gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/SshSession.java

@@ -42,11 +42,12 @@ public class SshSession {
   }
 
   @SuppressWarnings("resource")
-  public String exec(String command) throws JSchException, IOException {
+  public String exec(String command, InputStream opt) throws JSchException,
+      IOException {
     ChannelExec channel = (ChannelExec) getSession().openChannel("exec");
     try {
       channel.setCommand(command);
-      channel.setInputStream(null);
+      channel.setInputStream(opt);
       InputStream in = channel.getInputStream();
       channel.connect();
 
@@ -60,6 +61,20 @@ public class SshSession {
     }
   }
 
+  public InputStream exec2(String command, InputStream opt) throws JSchException,
+      IOException {
+    ChannelExec channel = (ChannelExec) getSession().openChannel("exec");
+    channel.setCommand(command);
+    channel.setInputStream(opt);
+    InputStream in = channel.getInputStream();
+    channel.connect();
+    return in;
+  }
+
+  public String exec(String command) throws JSchException, IOException {
+    return exec(command, null);
+  }
+
   public boolean hasError() {
     return error != null;
   }

gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/ssh/BUCK

@@ -2,5 +2,6 @@ include_defs('//gerrit-acceptance-tests/tests.defs')
 
 acceptance_tests(
   srcs = glob(['*IT.java']),
+  deps = ['//lib/commons:compress'],
   labels = ['ssh'],
 )

gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/ssh/UploadArchiveIT.java

@@ -0,0 +1,127 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.acceptance.ssh;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import com.google.gerrit.acceptance.AbstractDaemonTest;
+import com.google.gerrit.acceptance.GerritConfig;
+import com.google.gerrit.acceptance.NoHttpd;
+import com.google.gerrit.acceptance.PushOneCommit;
+
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.eclipse.jgit.transport.PacketLineIn;
+import org.eclipse.jgit.transport.PacketLineOut;
+import org.eclipse.jgit.util.IO;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Set;
+import java.util.TreeSet;
+
+@NoHttpd
+public class UploadArchiveIT extends AbstractDaemonTest {
+
+  @Test
+  @GerritConfig(name = "download.archive", value = "off")
+  public void archiveFeatureOff() throws Exception {
+    archiveNotPermitted();
+  }
+
+  @Test
+  @GerritConfig(name = "download.archive", values = {"tar", "tbz2", "tgz", "txz"})
+  public void zipFormatDisabled() throws Exception {
+    archiveNotPermitted();
+  }
+
+  @Test
+  public void zipFormat() throws Exception {
+    PushOneCommit.Result r = createChange();
+    String abbreviated = r.getCommitId().abbreviate(8).name();
+    String c = command(r, abbreviated);
+
+    InputStream out =
+        sshSession.exec2("git-upload-archive " + project.get(),
+            argumentsToInputStream(c));
+
+    // Wrap with PacketLineIn to read ACK bytes from output stream
+    PacketLineIn in = new PacketLineIn(out);
+    String tmp = in.readString();
+    assertThat(tmp).isEqualTo("ACK");
+    tmp = in.readString();
+
+    // Skip length (4 bytes) + 1 byte
+    // to position the output stream to the raw zip stream
+    byte[] buffer = new byte[5];
+    IO.readFully(out, buffer, 0, 5);
+    Set<String> entryNames = new TreeSet<>();
+    try (ZipArchiveInputStream zip = new ZipArchiveInputStream(out)) {
+      ZipArchiveEntry zipEntry = zip.getNextZipEntry();
+      while (zipEntry != null) {
+        String name = zipEntry.getName();
+        entryNames.add(name);
+        zipEntry = zip.getNextZipEntry();
+      }
+    }
+
+    assertThat(entryNames.size()).isEqualTo(1);
+    assertThat(Iterables.getOnlyElement(entryNames)).isEqualTo(
+        String.format("%s/%s", abbreviated, PushOneCommit.FILE_NAME));
+  }
+
+  private String command(PushOneCommit.Result r, String abbreviated) {
+    String c = "-f=zip "
+        + "-9 "
+        + "--prefix=" + abbreviated + "/ "
+        + r.getCommit().name() + " "
+        + PushOneCommit.FILE_NAME;
+    return c;
+  }
+
+  private void archiveNotPermitted() throws Exception {
+    PushOneCommit.Result r = createChange();
+    String abbreviated = r.getCommitId().abbreviate(8).name();
+    String c = command(r, abbreviated);
+
+    InputStream out =
+        sshSession.exec2("git-upload-archive " + project.get(),
+            argumentsToInputStream(c));
+
+    // Wrap with PacketLineIn to read ACK bytes from output stream
+    PacketLineIn in = new PacketLineIn(out);
+    String tmp = in.readString();
+    assertThat(tmp).isEqualTo("ACK");
+    tmp = in.readString();
+    tmp = in.readString();
+    tmp = tmp.substring(1);
+    assertThat(tmp).isEqualTo("fatal: upload-archive not permitted");
+  }
+
+  private InputStream argumentsToInputStream(String c) throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    PacketLineOut pctOut = new PacketLineOut(out);
+    for (String arg : Splitter.on(' ').split(c)) {
+      pctOut.writeString("argument " + arg);
+    }
+    pctOut.end();
+    return new ByteArrayInputStream(out.toByteArray());
+  }
+}

gerrit-httpd/src/main/java/com/google/gerrit/httpd/GerritConfigProvider.java

@@ -16,6 +16,7 @@ package com.google.gerrit.httpd;
 
 import com.google.common.base.Function;
 import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.gerrit.common.data.GerritConfig;
@@ -134,8 +135,18 @@ class GerritConfigProvider implements Provider<GerritConfig> {
     config.setChangeUpdateDelay((int) ConfigUtil.getTimeUnit(
         cfg, "change", null, "updateDelay", 30, TimeUnit.SECONDS));
     config.setLargeChangeSize(cfg.getInt("change", "largeChange", 500));
+
+    // Zip is not supported because it may be interpreted by a Java plugin as a
+    // valid JAR file, whose code would have access to cookies on the domain.
     config.setArchiveFormats(Lists.newArrayList(Iterables.transform(
-        archiveFormats.getAllowed(),
+        Iterables.filter(
+            archiveFormats.getAllowed(),
+            new Predicate<ArchiveFormat>() {
+              @Override
+              public boolean apply(ArchiveFormat format) {
+                return (format != ArchiveFormat.ZIP);
+              }
+            }),
         new Function<ArchiveFormat, String>() {
           @Override
           public String apply(ArchiveFormat in) {

gerrit-server/BUCK

@@ -48,6 +48,7 @@ java_library(
     '//lib/antlr:java_runtime',
     '//lib/auto:auto-value',
     '//lib/commons:codec',
+    '//lib/commons:compress',
     '//lib/commons:dbcp',
     '//lib/commons:lang',
     '//lib/commons:net',

gerrit-server/src/main/java/com/google/gerrit/server/change/ArchiveFormat.java

@@ -19,14 +19,14 @@ import org.eclipse.jgit.archive.TarFormat;
 import org.eclipse.jgit.archive.Tbz2Format;
 import org.eclipse.jgit.archive.TgzFormat;
 import org.eclipse.jgit.archive.TxzFormat;
+import org.eclipse.jgit.archive.ZipFormat;
 
 public enum ArchiveFormat {
   TGZ("application/x-gzip", new TgzFormat()),
   TAR("application/x-tar", new TarFormat()),
   TBZ2("application/x-bzip2", new Tbz2Format()),
-  TXZ("application/x-xz", new TxzFormat());
-  // Zip is not supported because it may be interpreted by a Java plugin as a
-  // valid JAR file, whose code would have access to cookies on the domain.
+  TXZ("application/x-xz", new TxzFormat()),
+  ZIP("application/x-zip", new ZipFormat());
 
   private final ArchiveCommand.Format<?> format;
   private final String mimeType;

gerrit-server/src/main/java/com/google/gerrit/server/change/GetArchive.java

@@ -18,6 +18,7 @@ import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.gerrit.extensions.restapi.BadRequestException;
 import com.google.gerrit.extensions.restapi.BinaryResult;
+import com.google.gerrit.extensions.restapi.MethodNotAllowedException;
 import com.google.gerrit.extensions.restapi.RestReadView;
 import com.google.gerrit.server.config.ConfigUtil;
 import com.google.gerrit.server.config.GerritServerConfig;
@@ -78,6 +79,10 @@ public class GetArchive implements RestReadView<RevisionResource> {
     public Set<ArchiveFormat> getAllowed() {
       return allowed;
     }
+
+    public ImmutableMap<String, ArchiveFormat> getExtensions() {
+      return extensions;
+    }
   }
 
   private final GitRepositoryManager repoManager;
@@ -93,8 +98,8 @@ public class GetArchive implements RestReadView<RevisionResource> {
   }
 
   @Override
-  public BinaryResult apply(RevisionResource rsrc)
-      throws BadRequestException, IOException {
+  public BinaryResult apply(RevisionResource rsrc) throws BadRequestException,
+      IOException, MethodNotAllowedException {
     if (Strings.isNullOrEmpty(format)) {
       throw new BadRequestException("format is not specified");
     }
@@ -102,6 +107,9 @@ public class GetArchive implements RestReadView<RevisionResource> {
     if (f == null) {
       throw new BadRequestException("unknown archive format");
     }
+    if (f == ArchiveFormat.ZIP) {
+      throw new MethodNotAllowedException("zip format is disabled");
+    }
     boolean close = true;
     final Repository repo = repoManager
         .openRepository(rsrc.getControl().getProject().getNameKey());

gerrit-sshd/BUCK

@@ -28,6 +28,7 @@ java_library(
     '//lib/mina:core',
     '//lib/mina:sshd',
     '//lib/jgit:jgit',
+    '//lib/jgit:jgit-archive',
   ],
   provided_deps = [
     '//lib/bouncycastle:bcprov',

gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java

@@ -70,6 +70,8 @@ public class DefaultCommandModule extends CommandModule {
     // Honor the legacy hyphenated forms as aliases for the non-hyphenated forms
     command("git-upload-pack").to(Commands.key(git, "upload-pack"));
     command(git, "upload-pack").to(Upload.class);
+    command("git-upload-archive").to(Commands.key(git, "upload-archive"));
+    command(git, "upload-archive").to(UploadArchive.class);
     command("suexec").to(SuExec.class);
     listener().to(ShowCaches.StartupListener.class);
 

gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/UploadArchive.java

@@ -0,0 +1,226 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.sshd.commands;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.gerrit.reviewdb.server.ReviewDb;
+import com.google.gerrit.server.change.ArchiveFormat;
+import com.google.gerrit.server.change.GetArchive;
+import com.google.gerrit.sshd.AbstractGitCommand;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+
+import org.eclipse.jgit.api.ArchiveCommand;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.transport.PacketLineIn;
+import org.eclipse.jgit.transport.PacketLineOut;
+import org.eclipse.jgit.transport.SideBandOutputStream;
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Allows getting archives for Git repositories over SSH using the Git
+ * upload-archive protocol.
+ */
+public class UploadArchive extends AbstractGitCommand {
+  /**
+   * Options for parsing Git commands.
+   * <p>
+   * These options are not passed on command line, but received through input
+   * stream in pkt-line format.
+   */
+  static class Options {
+    @Option(name = "-f", aliases = {"--format"}, usage = "Format of the"
+        + " resulting archive: tar or zip... If this option is not given, and"
+        + " the output file is specified, the format is inferred from the"
+        + " filename if possible (e.g. writing to \"foo.zip\" makes the output"
+        + " to be in the zip format). Otherwise the output format is tar.")
+    private String format = "tar";
+
+    @Option(name = "--prefix",
+        usage = "Prepend <prefix>/ to each filename in the archive.")
+    private String prefix;
+
+    @Option(name = "-0", usage = "Store the files instead of deflating them.")
+    private boolean level0;
+    @Option(name = "-1")
+    private boolean level1;
+    @Option(name = "-2")
+    private boolean level2;
+    @Option(name = "-3")
+    private boolean level3;
+    @Option(name = "-4")
+    private boolean level4;
+    @Option(name = "-5")
+    private boolean level5;
+    @Option(name = "-6")
+    private boolean level6;
+    @Option(name = "-7")
+    private boolean level7;
+    @Option(name = "-8")
+    private boolean level8;
+    @Option(name = "-9", usage = "Highest and slowest compression level. You "
+        + "can specify any number from 1 to 9 to adjust compression speed and "
+        + "ratio.")
+    private boolean level9;
+
+    @Argument(index = 0, required = true, usage = "The tree or commit to "
+        + "produce an archive for.")
+    private String treeIsh = "master";
+
+    @Argument(index = 1, multiValued = true, usage =
+        "Without an optional path parameter, all files and subdirectories of "
+        + "the current working directory are included in the archive. If one "
+        + "or more paths are specified, only these are included.")
+    private List<String> path;
+  }
+
+  @Inject
+  private GetArchive.AllowedFormats allowedFormats;
+  @Inject
+  private Provider<ReviewDb> db;
+  private Options options = new Options();
+
+  /**
+   * Read and parse arguments from input stream.
+   * This method gets the arguments from input stream, in Pkt-line format,
+   * then parses them to fill the options object.
+   */
+  protected void readArguments() throws IOException, Failure {
+    String argCmd = "argument ";
+    List<String> args = Lists.newArrayList();
+
+    // Read arguments in Pkt-Line format
+    PacketLineIn packetIn = new PacketLineIn(in);
+    for (;;) {
+      String s = packetIn.readString();
+      if (s == PacketLineIn.END) {
+        break;
+      }
+      if (!s.startsWith(argCmd)) {
+        throw new Failure(1, "fatal: 'argument' token or flush expected");
+      }
+      String[] parts = s.substring(argCmd.length()).split("=", 2);
+      for(String p : parts) {
+        args.add(p);
+      }
+    }
+
+    try {
+      // Parse them into the 'options' field
+      CmdLineParser parser = new CmdLineParser(options);
+      parser.parseArgument(args);
+      if (options.path == null || Arrays.asList(".").equals(options.path)) {
+        options.path = Collections.emptyList();
+      }
+    } catch (CmdLineException e) {
+      throw new Failure(2, "fatal: unable to parse arguments, " + e);
+    }
+  }
+
+  @Override
+  protected void runImpl() throws IOException, Failure {
+    PacketLineOut packetOut = new PacketLineOut(out);
+    packetOut.setFlushOnEnd(true);
+    packetOut.writeString("ACK");
+    packetOut.end();
+
+    try {
+      // Parse Git arguments
+      readArguments();
+
+      ArchiveFormat f = allowedFormats.getExtensions().get("." + options.format);
+      if (f == null) {
+        throw new Failure(3, "fatal: upload-archive not permitted");
+      }
+
+      // Find out the object to get from the specified reference and paths
+      ObjectId treeId = repo.resolve(options.treeIsh);
+      if (treeId.equals(ObjectId.zeroId())) {
+        throw new Failure(4, "fatal: reference not found");
+      }
+
+      // Verify the user has permissions to read the specified reference
+      if (!projectControl.allRefsAreVisible() && !canRead(treeId)) {
+          throw new Failure(5, "fatal: cannot perform upload-archive operation");
+      }
+
+      try {
+        // The archive is sent in DATA sideband channel
+        SideBandOutputStream sidebandOut =
+            new SideBandOutputStream(SideBandOutputStream.CH_DATA,
+                SideBandOutputStream.MAX_BUF, out);
+        new ArchiveCommand(repo)
+            .setFormat(f.name())
+            .setFormatOptions(getFormatOptions(f))
+            .setTree(treeId)
+            .setPaths(options.path.toArray(new String[0]))
+            .setPrefix(options.prefix)
+            .setOutputStream(sidebandOut)
+            .call();
+        sidebandOut.flush();
+        sidebandOut.close();
+      } catch (GitAPIException e) {
+        throw new Failure(7, "fatal: git api exception, " + e);
+      }
+    } catch (Failure f) {
+      // Report the error in ERROR sideband channel
+      SideBandOutputStream sidebandError =
+          new SideBandOutputStream(SideBandOutputStream.CH_ERROR,
+              SideBandOutputStream.MAX_BUF, out);
+      sidebandError.write(f.getMessage().getBytes(UTF_8));
+      sidebandError.flush();
+      sidebandError.close();
+      throw f;
+    } finally {
+      // In any case, cleanly close the packetOut channel
+      packetOut.end();
+    }
+  }
+
+  private Map<String, Object> getFormatOptions(ArchiveFormat f) {
+    if (f == ArchiveFormat.ZIP) {
+      int value = Arrays.asList(options.level0, options.level1, options.level2,
+          options.level3, options.level4, options.level5, options.level6,
+          options.level7, options.level8, options.level9).indexOf(true);
+      if (value >= 0) {
+        return ImmutableMap.<String, Object> of(
+            "level", Integer.valueOf(value));
+      }
+    }
+    return Collections.emptyMap();
+  }
+
+  private boolean canRead(ObjectId revId) throws IOException {
+    try (RevWalk rw = new RevWalk(repo)) {
+      RevCommit commit = rw.parseCommit(revId);
+      return projectControl.canReadCommit(db.get(), rw, commit);
+    }
+  }
+}

lib/commons/BUCK

@@ -23,7 +23,6 @@ maven_jar(
   sha1 = 'ab365c96ee9bc88adcc6fa40d185c8e15a31410d',
   license = 'Apache2.0',
   exclude = ['META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'],
-  visibility = ['//lib/jgit:jgit-archive'],
 )
 
 maven_jar(