Merge branch 'stable-2.16' into stable-3.0

* stable-2.16:
  Update git submodules
  Update git submodules
  Apply diff preferences immediately after clicking save
  Grant the InternalUser back access to changes
  Rewrite upload archive tests as real integration tests
  Add support for Elasticsearch version 7.7.*

Change-Id: I531f7336bdbe191b5d88bbc1282a59861250c897

javatests/com/google/gerrit/acceptance/pgm/ElasticReindexIT.java

@@ -32,7 +32,7 @@ public class ElasticReindexIT extends AbstractReindexTests {
 
   @ConfigSuite.Config
   public static Config elasticsearchV7() {
-    return getConfig(ElasticVersion.V7_6);
+    return getConfig(ElasticVersion.V7_7);
   }
 
   @Override

javatests/com/google/gerrit/acceptance/ssh/ElasticIndexIT.java

@@ -31,7 +31,7 @@ public class ElasticIndexIT extends AbstractIndexTests {
 
   @ConfigSuite.Config
   public static Config elasticsearchV7() {
-    return getConfig(ElasticVersion.V7_6);
+    return getConfig(ElasticVersion.V7_7);
   }
 
   @Override

javatests/com/google/gerrit/acceptance/ssh/UploadArchiveIT.java

@@ -1,158 +0,0 @@
-// Copyright (C) 2015 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.gerrit.acceptance.ssh;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.common.base.Splitter;
-import com.google.gerrit.acceptance.AbstractDaemonTest;
-import com.google.gerrit.acceptance.GerritConfig;
-import com.google.gerrit.acceptance.NoHttpd;
-import com.google.gerrit.acceptance.PushOneCommit;
-import com.google.gerrit.acceptance.UseSsh;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
-import java.util.Set;
-import java.util.TreeSet;
-import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
-import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
-import org.eclipse.jgit.transport.PacketLineIn;
-import org.eclipse.jgit.transport.PacketLineOut;
-import org.eclipse.jgit.util.IO;
-import org.junit.Test;
-
-@NoHttpd
-@UseSsh
-public class UploadArchiveIT extends AbstractDaemonTest {
-
-  @Test
-  @GerritConfig(name = "download.archive", value = "off")
-  public void archiveFeatureOff() throws Exception {
-    assertArchiveNotPermitted();
-  }
-
-  @Test
-  @GerritConfig(
-      name = "download.archive",
-      values = {"tar", "tbz2", "tgz", "txz"})
-  public void zipFormatDisabled() throws Exception {
-    assertArchiveNotPermitted();
-  }
-
-  @Test
-  public void zipFormat() throws Exception {
-    PushOneCommit.Result r = createChange();
-    String abbreviated = r.getCommit().abbreviate(8).name();
-    String c = command(r, "zip", abbreviated);
-
-    InputStream out =
-        adminSshSession.exec2("git-upload-archive " + project.get(), argumentsToInputStream(c));
-
-    // Wrap with PacketLineIn to read ACK bytes from output stream
-    PacketLineIn in = new PacketLineIn(out);
-    String tmp = in.readString();
-    assertThat(tmp).isEqualTo("ACK");
-    in.readString();
-
-    // Skip length (4 bytes) + 1 byte
-    // to position the output stream to the raw zip stream
-    byte[] buffer = new byte[5];
-    IO.readFully(out, buffer, 0, 5);
-    Set<String> entryNames = new TreeSet<>();
-    try (ZipArchiveInputStream zip = new ZipArchiveInputStream(out)) {
-      ZipArchiveEntry zipEntry = zip.getNextZipEntry();
-      while (zipEntry != null) {
-        String name = zipEntry.getName();
-        entryNames.add(name);
-        zipEntry = zip.getNextZipEntry();
-      }
-    }
-
-    assertThat(entryNames)
-        .containsExactly(
-            String.format("%s/", abbreviated),
-            String.format("%s/%s", abbreviated, PushOneCommit.FILE_NAME))
-        .inOrder();
-  }
-
-  // Make sure we have coverage for the dependency on xz.
-  @Test
-  public void txzFormat() throws Exception {
-    PushOneCommit.Result r = createChange();
-    String abbreviated = r.getCommit().abbreviate(8).name();
-    String c = command(r, "tar.xz", abbreviated);
-
-    try (InputStream out =
-        adminSshSession.exec2("git-upload-archive " + project.get(), argumentsToInputStream(c))) {
-
-      // Wrap with PacketLineIn to read ACK bytes from output stream
-      PacketLineIn in = new PacketLineIn(out);
-      String packet = in.readString();
-      assertThat(packet).isEqualTo("ACK");
-
-      // Discard first bit of data, which should be empty.
-      packet = in.readString();
-      assertThat(packet).isEmpty();
-
-      // Make sure the next one is not on the error channel
-      packet = in.readString();
-
-      // 1 = DATA. It would be nicer to parse the OutputStream with SideBandInputStream from JGit,
-      // but
-      // that is currently not public.
-      char channel = packet.charAt(0);
-      if (channel != 1) {
-        fail("got packet on channel " + (int) channel, packet);
-      }
-    }
-  }
-
-  private String command(PushOneCommit.Result r, String format, String abbreviated) {
-    String c =
-        String.format(
-            "-f=%s --prefix=%s/ %s %s",
-            format, abbreviated, r.getCommit().name(), PushOneCommit.FILE_NAME);
-    return c;
-  }
-
-  private void assertArchiveNotPermitted() throws Exception {
-    PushOneCommit.Result r = createChange();
-    String abbreviated = r.getCommit().abbreviate(8).name();
-    String c = command(r, "zip", abbreviated);
-
-    InputStream out =
-        adminSshSession.exec2("git-upload-archive " + project.get(), argumentsToInputStream(c));
-
-    // Wrap with PacketLineIn to read ACK bytes from output stream
-    PacketLineIn in = new PacketLineIn(out);
-    String tmp = in.readString();
-    assertThat(tmp).isEqualTo("ACK");
-    in.readString();
-    tmp = in.readString();
-    tmp = tmp.substring(1);
-    assertThat(tmp).isEqualTo("fatal: upload-archive not permitted for format zip");
-  }
-
-  private InputStream argumentsToInputStream(String c) throws Exception {
-    ByteArrayOutputStream out = new ByteArrayOutputStream();
-    PacketLineOut pctOut = new PacketLineOut(out);
-    for (String arg : Splitter.on(' ').split(c)) {
-      pctOut.writeString("argument " + arg);
-    }
-    pctOut.end();
-    return new ByteArrayInputStream(out.toByteArray());
-  }
-}

javatests/com/google/gerrit/elasticsearch/ElasticContainer.java

@@ -58,6 +58,8 @@ public class ElasticContainer extends ElasticsearchContainer {
         return "blacktop/elasticsearch:7.5.2";
       case V7_6:
         return "blacktop/elasticsearch:7.6.2";
+      case V7_7:
+        return "blacktop/elasticsearch:7.7.0";
     }
     throw new IllegalStateException("No tests for version: " + version.name());
   }

javatests/com/google/gerrit/elasticsearch/ElasticV7QueryAccountsTest.java

@@ -36,7 +36,7 @@ public class ElasticV7QueryAccountsTest extends AbstractQueryAccountsTest {
   public static void startIndexService() {
     if (container == null) {
       // Only start Elasticsearch once
-      container = ElasticContainer.createAndStart(ElasticVersion.V7_6);
+      container = ElasticContainer.createAndStart(ElasticVersion.V7_7);
     }
   }
 

javatests/com/google/gerrit/elasticsearch/ElasticV7QueryChangesTest.java

@@ -42,7 +42,7 @@ public class ElasticV7QueryChangesTest extends AbstractQueryChangesTest {
   public static void startIndexService() {
     if (container == null) {
       // Only start Elasticsearch once
-      container = ElasticContainer.createAndStart(ElasticVersion.V7_6);
+      container = ElasticContainer.createAndStart(ElasticVersion.V7_7);
       client = HttpAsyncClients.createDefault();
       client.start();
     }

javatests/com/google/gerrit/elasticsearch/ElasticV7QueryGroupsTest.java

@@ -36,7 +36,7 @@ public class ElasticV7QueryGroupsTest extends AbstractQueryGroupsTest {
   public static void startIndexService() {
     if (container == null) {
       // Only start Elasticsearch once
-      container = ElasticContainer.createAndStart(ElasticVersion.V7_6);
+      container = ElasticContainer.createAndStart(ElasticVersion.V7_7);
     }
   }
 

javatests/com/google/gerrit/elasticsearch/ElasticV7QueryProjectsTest.java

@@ -36,7 +36,7 @@ public class ElasticV7QueryProjectsTest extends AbstractQueryProjectsTest {
   public static void startIndexService() {
     if (container == null) {
       // Only start Elasticsearch once
-      container = ElasticContainer.createAndStart(ElasticVersion.V7_6);
+      container = ElasticContainer.createAndStart(ElasticVersion.V7_7);
     }
   }
 

javatests/com/google/gerrit/elasticsearch/ElasticVersionTest.java

@@ -51,6 +51,9 @@ public class ElasticVersionTest extends GerritBaseTests {
 
     assertThat(ElasticVersion.forVersion("7.6.0")).isEqualTo(ElasticVersion.V7_6);
     assertThat(ElasticVersion.forVersion("7.6.1")).isEqualTo(ElasticVersion.V7_6);
+
+    assertThat(ElasticVersion.forVersion("7.7.0")).isEqualTo(ElasticVersion.V7_7);
+    assertThat(ElasticVersion.forVersion("7.7.1")).isEqualTo(ElasticVersion.V7_7);
   }
 
   @Test
@@ -73,6 +76,7 @@ public class ElasticVersionTest extends GerritBaseTests {
     assertThat(ElasticVersion.V7_4.isAtLeastMinorVersion(ElasticVersion.V6_7)).isFalse();
     assertThat(ElasticVersion.V7_5.isAtLeastMinorVersion(ElasticVersion.V6_7)).isFalse();
     assertThat(ElasticVersion.V7_6.isAtLeastMinorVersion(ElasticVersion.V6_7)).isFalse();
+    assertThat(ElasticVersion.V7_7.isAtLeastMinorVersion(ElasticVersion.V6_7)).isFalse();
   }
 
   @Test
@@ -87,6 +91,7 @@ public class ElasticVersionTest extends GerritBaseTests {
     assertThat(ElasticVersion.V7_4.isV6OrLater()).isTrue();
     assertThat(ElasticVersion.V7_5.isV6OrLater()).isTrue();
     assertThat(ElasticVersion.V7_6.isV6OrLater()).isTrue();
+    assertThat(ElasticVersion.V7_7.isV6OrLater()).isTrue();
   }
 
   @Test
@@ -101,5 +106,6 @@ public class ElasticVersionTest extends GerritBaseTests {
     assertThat(ElasticVersion.V7_4.isV7OrLater()).isTrue();
     assertThat(ElasticVersion.V7_5.isV7OrLater()).isTrue();
     assertThat(ElasticVersion.V7_6.isV7OrLater()).isTrue();
+    assertThat(ElasticVersion.V7_7.isV7OrLater()).isTrue();
   }
 }

javatests/com/google/gerrit/integration/git/BUILD

@@ -0,0 +1,7 @@
+load("//javatests/com/google/gerrit/acceptance:tests.bzl", "acceptance_tests")
+
+acceptance_tests(
+    srcs = ["UploadArchiveIT.java"],
+    group = "upload-archive",
+    labels = ["git-upload-archive"],
+)

javatests/com/google/gerrit/integration/git/UploadArchiveIT.java

@@ -0,0 +1,211 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.gerrit.integration.git;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.acceptance.PushOneCommit.FILE_CONTENT;
+import static com.google.gerrit.acceptance.PushOneCommit.FILE_NAME;
+import static com.google.gerrit.testing.GerritJUnit.assertThrows;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.gerrit.acceptance.GerritConfig;
+import com.google.gerrit.acceptance.GerritServer.TestSshServerAddress;
+import com.google.gerrit.acceptance.NoHttpd;
+import com.google.gerrit.acceptance.StandaloneSiteTest;
+import com.google.gerrit.acceptance.UseSsh;
+import com.google.gerrit.common.RawInputUtil;
+import com.google.gerrit.extensions.api.GerritApi;
+import com.google.gerrit.extensions.common.ChangeInput;
+import com.google.gerrit.extensions.common.CommitInfo;
+import com.google.gerrit.extensions.restapi.RestApiException;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.Inject;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
+import org.junit.Test;
+
+@NoHttpd
+@UseSsh
+public class UploadArchiveIT extends StandaloneSiteTest {
+  private static final String[] SSH_KEYGEN_CMD =
+      new String[] {"ssh-keygen", "-t", "rsa", "-q", "-P", "", "-f"};
+  private static final String GIT_SSH_COMMAND =
+      "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o 'IdentitiesOnly yes' -i";
+  private static final String ARCHIVE = "archive";
+
+  @Inject private GerritApi gApi;
+  @Inject private @TestSshServerAddress InetSocketAddress sshAddress;
+
+  private String sshDestination;
+  private String identityPath;
+  private Project.NameKey project;
+  private CommitInfo commit;
+
+  @Test
+  @GerritConfig(name = "download.archive", value = "off")
+  public void archiveFeatureOff() throws Exception {
+    try (ServerContext ctx = startServer()) {
+      setUpTestHarness(ctx);
+      assertArchiveNotPermitted();
+    }
+  }
+
+  @Test
+  @GerritConfig(
+      name = "download.archive",
+      values = {"tar", "tbz2", "tgz", "txz"})
+  public void zipFormatDisabled() throws Exception {
+    try (ServerContext ctx = startServer()) {
+      setUpTestHarness(ctx);
+      assertArchiveNotPermitted();
+    }
+  }
+
+  @Test
+  public void verifyUploadArchiveFormats() throws Exception {
+    try (ServerContext ctx = startServer()) {
+      setUpTestHarness(ctx);
+      setUpChange();
+      for (String f : Arrays.asList("zip", "tar", "tar.gz", "tar.bz2", "tar.xz")) {
+        verifyUploadArchive(f);
+      }
+    }
+  }
+
+  private void verifyUploadArchive(String format) throws Exception {
+    Path outputPath = sitePaths.data_dir.resolve(ARCHIVE);
+    execute(
+        cmd(format, commit.commit),
+        sitePaths.data_dir.toFile(),
+        ImmutableMap.of("GIT_SSH_COMMAND", GIT_SSH_COMMAND + identityPath),
+        outputPath);
+    try (InputStream fi = Files.newInputStream(outputPath);
+        InputStream bi = new BufferedInputStream(fi);
+        ArchiveInputStream archive = archiveStreamForFormat(bi, format)) {
+      assertEntries(archive);
+    }
+  }
+
+  private ArchiveInputStream archiveStreamForFormat(InputStream bi, String format)
+      throws IOException {
+    switch (format) {
+      case "zip":
+        return new ZipArchiveInputStream(bi);
+      case "tar":
+        return new TarArchiveInputStream(bi);
+      case "tar.gz":
+        return new TarArchiveInputStream(new GzipCompressorInputStream(bi));
+      case "tar.bz2":
+        return new TarArchiveInputStream(new BZip2CompressorInputStream(bi));
+      case "tar.xz":
+        return new TarArchiveInputStream(new XZCompressorInputStream(bi));
+      default:
+        throw new IllegalArgumentException("Unknown archive format: " + format);
+    }
+  }
+
+  private void setUpTestHarness(ServerContext ctx) throws RestApiException, Exception {
+    ctx.getInjector().injectMembers(this);
+    project = new Project.NameKey("upload-archive-project-test");
+    gApi.projects().create(project.get());
+    setUpAuthentication();
+    sshDestination =
+        String.format(
+            "ssh://%s@%s:%s/%s",
+            "admin", sshAddress.getHostName(), sshAddress.getPort(), project.get());
+    identityPath = sitePaths.data_dir.resolve(String.format("id_rsa_%s", "admin")).toString();
+  }
+
+  private void setUpAuthentication() throws Exception {
+    execute(
+        ImmutableList.<String>builder()
+            .add(SSH_KEYGEN_CMD)
+            .add(String.format("id_rsa_%s", "admin"))
+            .build());
+    gApi.accounts()
+        .id("admin")
+        .addSshKey(
+            new String(
+                java.nio.file.Files.readAllBytes(
+                    sitePaths.data_dir.resolve(String.format("id_rsa_%s.pub", "admin"))),
+                UTF_8));
+  }
+
+  private ImmutableList<String> cmd(String format, String commit) {
+    return ImmutableList.<String>builder()
+        .add("git")
+        .add("archive")
+        .add("-f=" + format)
+        .add("--prefix=" + commit + "/")
+        .add("--remote=" + sshDestination)
+        .add(commit)
+        .add(FILE_NAME)
+        .build();
+  }
+
+  private String execute(ImmutableList<String> cmd) throws Exception {
+    return execute(cmd, sitePaths.data_dir.toFile(), ImmutableMap.of());
+  }
+
+  private void assertArchiveNotPermitted() {
+    IOException exception =
+        assertThrows(
+            IOException.class,
+            () ->
+                execute(
+                    cmd("zip", "master"),
+                    sitePaths.data_dir.toFile(),
+                    ImmutableMap.of("GIT_SSH_COMMAND", GIT_SSH_COMMAND + identityPath)));
+    assertThat(exception)
+        .hasMessageThat()
+        .contains("fatal: upload-archive not permitted for format zip");
+  }
+
+  private void setUpChange() throws Exception {
+    ChangeInput in = new ChangeInput(project.get(), "master", "Test change");
+    in.newBranch = true;
+    String changeId = gApi.changes().create(in).info().changeId;
+    gApi.changes().id(changeId).edit().modifyFile(FILE_NAME, RawInputUtil.create(FILE_CONTENT));
+    gApi.changes().id(changeId).edit().publish();
+    commit = gApi.changes().id(changeId).current().commit(false);
+  }
+
+  private void assertEntries(ArchiveInputStream o) throws IOException {
+    Set<String> entryNames = new TreeSet<>();
+    ArchiveEntry e;
+    while ((e = o.getNextEntry()) != null) {
+      entryNames.add(e.getName());
+    }
+    assertThat(entryNames)
+        .containsExactly(
+            String.format("%s/", commit.commit), String.format("%s/%s", commit.commit, FILE_NAME))
+        .inOrder();
+  }
+}

javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java

@@ -1881,6 +1881,11 @@ public abstract class AbstractQueryChangesTest extends GerritServerTests {
         new AccountGroup.UUID(gApi.groups().id(g1).get().id));
     assertQuery(q + " visibleto:" + g1, change1);
 
+    // Both changes are visible to InternalUser
+    try (ManualRequestContext ctx = oneOffRequestContext.open()) {
+      assertQuery(q, change2, change1);
+    }
+
     requestContext.setContext(newRequestContext(user2));
     assertQuery("is:visible", change1);