Merge branch 'stable-3.2' into stable-3.3

* stable-3.2:
  Set version to 3.2.5.1
  Disk cache metrics require cache.enableDiskStatMetrics
  Set version to 2.14.22
  ElasticContainer: Upgrade V6_8 to elasticsearch 6.8.13
  Upgrade testcontainers to 1.15.0
  Update git submodules
  Workaround Gitiles bug on All-Users visibility
  Validate Gerrit changes on stable-2.15 with Jenkins
  Set version to 2.15.22-SNAPSHOT
  Set version to 3.0.16-SNAPSHOT
  Set version to 2.15.21
  Set version to 3.0.15
  Set version to 2.16.26-SNAPSHOT
  Set version to 2.16.25
  Set version to 3.1.11-SNAPSHOT
  Set version to 3.1.10
  Workaround Gitiles bug on All-Users visibility
  Workaround Gitiles bug on All-Users visibility
  Workaround Gitiles bug on All-Users visibility
  Workaround Gitiles bug on All-Users visibility
  Set version to 3.2.6-SNAPSHOT
  Set version to 3.2.5
  Workaround Gitiles bug on All-Users visibility
  Set version to 2.15.21-SNAPSHOT
  Set version to 2.15.20
  Fetch JGit documentation from the archive site
  Remove generation for c.g.gwtexpui.* JavaDoc
  Set version to 3.2.5-SNAPSHOT
  Set version to 3.2.4
  Set version to 3.1.10-SNAPSHOT
  Set version to 3.1.9
  Set version to 3.0.15-SNAPSHOT
  Set version to 2.16.25-SNAPSHOT
  Set version to 3.0.14
  Set version to 2.16.24
  Make PermissionBackend#ForRef authoritative
  Validate Gerrit changes on stable-2.15 with Jenkins
  Fix tests for stable-2.15 branch
  Make PermissionBackend#ForRef authoritative
  Make PermissionBackend#ForRef authoritative
  Make PermissionBackend#ForRef authoritative
  Verify hostname when sending emails via SMTP server with SMTPSClient
  Make PermissionBackend#ForRef authoritative
  Docs: Clarify that 'm' push option sets patch set description

Change-Id: I5f9d527fc82a92cfd1ee5aead34532c2054e9d81

Documentation/concept-patch-sets.txt

@@ -89,7 +89,7 @@ evolves, such as "Added more unit tests." Unlike the change description, a patch
 set description does not become a part of the project's history.
 
 To add a patch set description, click *Add a patch set description*, located in
-the file list.
+the file list, or provide it link:user-upload.html#patch_set_description[on upload].
 
 GERRIT
 ------

Documentation/metrics.txt

@@ -66,6 +66,11 @@ objects needing finalization.
 * `caches/disk_hit_ratio`: Disk hit ratio for persistent cache.
 * `caches/refresh_count`: The number of refreshes per cache with an indicator if a reload was necessary.
 
+Cache disk metrics are expensive to compute on larger installations and are not
+computed by default. They can be enabled via the
+link:config.gerrit.html#cache.enableDiskStatMetrics[`cache.enableDiskStatMetrics`]
+setting.
+
 === Change
 
 * `change/submit_rule_evaluation`: Latency for evaluating submit rules on a change.

Documentation/user-upload.txt

@@ -315,11 +315,11 @@ link:intro-user.html#work-in-progress-by-default[user preference]. If the
 preference is set so the default behavior is to create `work-in-progress`
 changes, this can be overridden with the `ready` option.
 
-[[message]]
-==== Message
+[[patch_set_description]]
+==== Patch Set Description
 
-A comment message can be applied to the change by using the `message` (or `m`)
-option:
+A link:concept-patch-sets.html#_description[patch set description] can be
+applied by using the `message` (or `m`) option:
 
 ----
   git push ssh://john.doe@git.example.com:29418/kernel/common HEAD:refs/for/experimental%m=This_is_a_rebase_on_master%21

java/com/google/gerrit/server/mail/send/SmtpEmailSender.java

@@ -391,11 +391,7 @@ public class SmtpEmailSender implements EmailSender {
   }
 
   private SMTPClient open() throws EmailException {
-    final AuthSMTPClient client = new AuthSMTPClient(UTF_8.name());
-
-    if (smtpEncryption == Encryption.SSL) {
-      client.enableSSL(sslVerify);
-    }
+    final AuthSMTPClient client = new AuthSMTPClient(smtpEncryption == Encryption.SSL, sslVerify);
 
     client.setConnectTimeout(connectTimeout);
     try {
@@ -411,7 +407,7 @@ public class SmtpEmailSender implements EmailSender {
       }
 
       if (smtpEncryption == Encryption.TLS) {
-        if (!client.startTLS(smtpHost, smtpPort, sslVerify)) {
+        if (!client.execTLS()) {
           throw new EmailException("SMTP server does not support TLS");
         }
         if (!client.login()) {

java/com/google/gerrit/server/permissions/ProjectControl.java

@@ -36,6 +36,7 @@ import com.google.gerrit.extensions.conditions.BooleanCondition;
 import com.google.gerrit.extensions.restapi.AuthException;
 import com.google.gerrit.server.CurrentUser;
 import com.google.gerrit.server.account.GroupMembership;
+import com.google.gerrit.server.config.AllUsersName;
 import com.google.gerrit.server.config.GitReceivePackGroups;
 import com.google.gerrit.server.config.GitUploadPackGroups;
 import com.google.gerrit.server.git.GitRepositoryManager;
@@ -76,6 +77,7 @@ class ProjectControl {
   private final PermissionCollection.Factory permissionFilter;
   private final DefaultRefFilter.Factory refFilterFactory;
   private final ChangeData.Factory changeDataFactory;
+  private final AllUsersName allUsersName;
 
   private List<SectionMatcher> allSections;
   private Map<String, RefControl> refControls;
@@ -91,6 +93,7 @@ class ProjectControl {
       GitRepositoryManager repositoryManager,
       DefaultRefFilter.Factory refFilterFactory,
       ChangeData.Factory changeDataFactory,
+      AllUsersName allUsersName,
       @Assisted CurrentUser who,
       @Assisted ProjectState ps) {
     this.uploadGroups = uploadGroups;
@@ -101,6 +104,7 @@ class ProjectControl {
     this.repositoryManager = repositoryManager;
     this.refFilterFactory = refFilterFactory;
     this.changeDataFactory = changeDataFactory;
+    this.allUsersName = allUsersName;
     user = who;
     state = ps;
   }
@@ -173,7 +177,9 @@ class ProjectControl {
   }
 
   boolean allRefsAreVisible(Set<String> ignore) {
-    return user.isInternalUser() || canPerformOnAllRefs(Permission.READ, ignore);
+    return user.isInternalUser()
+        || (!getProject().getNameKey().equals(allUsersName)
+            && canPerformOnAllRefs(Permission.READ, ignore));
   }
 
   /** Can the user run upload pack? */

java/com/google/gerrit/util/ssl/BlindSSLSocketFactory.java

@@ -20,7 +20,6 @@ import java.net.Socket;
 import java.net.UnknownHostException;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
-import java.security.cert.X509Certificate;
 import javax.net.SocketFactory;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLSocketFactory;
@@ -32,19 +31,7 @@ public class BlindSSLSocketFactory extends SSLSocketFactory {
   private static final BlindSSLSocketFactory INSTANCE;
 
   static {
-    final X509TrustManager dummyTrustManager =
-        new X509TrustManager() {
-          @Override
-          public X509Certificate[] getAcceptedIssuers() {
-            return null;
-          }
-
-          @Override
-          public void checkClientTrusted(X509Certificate[] chain, String authType) {}
-
-          @Override
-          public void checkServerTrusted(X509Certificate[] chain, String authType) {}
-        };
+    final X509TrustManager dummyTrustManager = new BlindTrustManager();
 
     try {
       final SSLContext context = SSLContext.getInstance("SSL");

java/com/google/gerrit/util/ssl/BlindTrustManager.java

@@ -0,0 +1,33 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.util.ssl;
+
+import java.security.cert.X509Certificate;
+import javax.net.ssl.X509TrustManager;
+
+/** TrustManager implementation that accepts all certificates without validation. */
+public class BlindTrustManager implements X509TrustManager {
+
+  @Override
+  public X509Certificate[] getAcceptedIssuers() {
+    return null;
+  }
+
+  @Override
+  public void checkClientTrusted(X509Certificate[] chain, String authType) {}
+
+  @Override
+  public void checkServerTrusted(X509Certificate[] chain, String authType) {}
+}

java/org/apache/commons/net/smtp/AuthSMTPClient.java

@@ -17,68 +17,66 @@ package org.apache.commons.net.smtp;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.common.io.BaseEncoding;
-import com.google.gerrit.util.ssl.BlindSSLSocketFactory;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
+import com.google.gerrit.util.ssl.BlindTrustManager;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
 import java.io.UnsupportedEncodingException;
-import java.net.SocketException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
 import java.util.List;
 import javax.crypto.Mac;
 import javax.crypto.spec.SecretKeySpec;
-import javax.net.ssl.SSLParameters;
-import javax.net.ssl.SSLSocket;
-import javax.net.ssl.SSLSocketFactory;
 
-public class AuthSMTPClient extends SMTPClient {
+/**
+ * SMTP Client with authentication support and optional SSL processing and verification. {@link
+ * org.apache.commons.net.smtp.SMTPSClient} is used for the SSL handshake and hostname verification.
+ *
+ * <p>If shouldHandshakeOnConnect mode is selected, SSL/TLS negotiation starts right after the
+ * connection has been established. Otherwise SSL/TLS negotiation will only occur if {@link
+ * AuthSMTPClient#execTLS} is explicitly called and the server accepts the command.
+ *
+ * <p>Examples:
+ *
+ * <ul>
+ *   <li>For SSL connection:
+ *       <pre>
+ *       AuthSMTPClient c = new AuthSMTPClient(true, sslVerify);
+ *       c.connect("127.0.0.1", 465);
+ *     </pre>
+ *   <li>For TLS connection:
+ *       <pre>
+ *       AuthSMTPClient c = new AuthSMTPClient(false, sslVerify);
+ *       c.connect("127.0.0.1", 25);
+ *       if (c.execTLS()) { /rest of the commands here/ }
+ *     </pre>
+ *   <li>If SSL encryption is not required:
+ *       <pre>
+ *       AuthSMTPClient c = new AuthSMTPClient(false, false);
+ *       c.connect("127.0.0.1", port);
+ *     </pre>
+ */
+public class AuthSMTPClient extends SMTPSClient {
+
   private String authTypes;
 
-  public AuthSMTPClient(String charset) {
-    super(charset);
+  /**
+   * Constructs AuthSMTPClient.
+   *
+   * @param shouldHandshakeOnConnect the SSL processing mode, {@code true} if SSL negotiation should
+   *     start right after connect, {@code false} if it will be started by the user explicitly or
+   *     SSL negotiation is not required.
+   * @param sslVerificationEnabled {@code true} if the SMTP server's SSL certificate and hostname
+   *     should be verified, {@code false} otherwise.
+   */
+  public AuthSMTPClient(boolean shouldHandshakeOnConnect, boolean sslVerificationEnabled) {
+    // If SSL Encryption is required, SMTPSClient is used for the handshake.
+    // Otherwise, use  SMTPSClient in 'explicit' mode without calling execTLS().
+    // See SMTPSClient._connectAction_ in commons-net-3.6.
+    super("TLS", shouldHandshakeOnConnect, UTF_8.name());
+    this.setEndpointCheckingEnabled(sslVerificationEnabled);
+    if (!sslVerificationEnabled) {
+      this.setTrustManager(new BlindTrustManager());
     }
-
-  public void enableSSL(boolean verify) {
-    _socketFactory_ = sslFactory(verify);
-  }
-
-  public boolean startTLS(String hostname, int port, boolean verify)
-      throws SocketException, IOException {
-    if (sendCommand("STARTTLS") != 220) {
-      return false;
-    }
-
-    _socket_ = sslFactory(verify).createSocket(_socket_, hostname, port, true);
-
-    if (verify) {
-      SSLParameters sslParams = new SSLParameters();
-      sslParams.setEndpointIdentificationAlgorithm("HTTPS");
-      ((SSLSocket) _socket_).setSSLParameters(sslParams);
-    }
-
-    // XXX: Can't call _connectAction_() because SMTP server doesn't
-    // give banner information again after STARTTLS, thus SMTP._connectAction_()
-    // will wait on __getReply() forever, see source code of commons-net-2.2.
-    //
-    // The lines below are copied from SocketClient._connectAction_() and
-    // SMTP._connectAction_() in commons-net-2.2.
-    _socket_.setSoTimeout(_timeout_);
-    _input_ = _socket_.getInputStream();
-    _output_ = _socket_.getOutputStream();
-    _reader = new BufferedReader(new InputStreamReader(_input_, UTF_8));
-    _writer = new BufferedWriter(new OutputStreamWriter(_output_, UTF_8));
-    return true;
-  }
-
-  private static SSLSocketFactory sslFactory(boolean verify) {
-    if (verify) {
-      return (SSLSocketFactory) SSLSocketFactory.getDefault();
-    }
-    return (SSLSocketFactory) BlindSSLSocketFactory.getDefault();
   }
 
   @Override

javatests/com/google/gerrit/elasticsearch/BUILD

@@ -17,8 +17,11 @@ java_library(
         "//lib:junit",
         "//lib/guice",
         "//lib/httpcomponents:httpcore",
+        "//lib/jackson:jackson-annotations",
         "//lib/log:api",
         "//lib/testcontainers",
+        "//lib/testcontainers:docker-java-api",
+        "//lib/testcontainers:docker-java-transport",
         "//lib/testcontainers:testcontainers-elasticsearch",
     ],
 )

javatests/com/google/gerrit/elasticsearch/ElasticContainer.java

@@ -19,6 +19,7 @@ import org.junit.AssumptionViolatedException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.elasticsearch.ElasticsearchContainer;
+import org.testcontainers.utility.DockerImageName;
 
 /* Helper class for running ES integration tests in docker container */
 public class ElasticContainer extends ElasticsearchContainer {
@@ -39,7 +40,7 @@ public class ElasticContainer extends ElasticsearchContainer {
   private static String getImageName(ElasticVersion version) {
     switch (version) {
       case V6_8:
-        return "blacktop/elasticsearch:6.8.12";
+        return "blacktop/elasticsearch:6.8.13";
       case V7_0:
         return "blacktop/elasticsearch:7.0.1";
       case V7_1:
@@ -63,7 +64,9 @@ public class ElasticContainer extends ElasticsearchContainer {
   }
 
   private ElasticContainer(ElasticVersion version) {
-    super(getImageName(version));
+    super(
+        DockerImageName.parse(getImageName(version))
+            .asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"));
   }
 
   @Override

javatests/com/google/gerrit/server/permissions/RefControlTest.java

@@ -48,6 +48,7 @@ import com.google.gerrit.server.CurrentUser;
 import com.google.gerrit.server.account.GroupMembership;
 import com.google.gerrit.server.account.ListGroupMembership;
 import com.google.gerrit.server.config.AllProjectsName;
+import com.google.gerrit.server.config.AllUsersName;
 import com.google.gerrit.server.git.meta.MetaDataUpdate;
 import com.google.gerrit.server.index.SingleVersionModule.SingleVersionListener;
 import com.google.gerrit.server.project.ProjectCache;
@@ -63,6 +64,7 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Optional;
 import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.lib.Repository;
@@ -90,6 +92,18 @@ public class RefControlTest {
     assertWithMessage("not owner").that(u.isOwner()).isFalse();
   }
 
+  private void assertAllRefsAreVisible(ProjectControl u) {
+    assertWithMessage("all refs visible")
+        .that(u.allRefsAreVisible(Collections.emptySet()))
+        .isTrue();
+  }
+
+  private void assertAllRefsAreNotVisible(ProjectControl u) {
+    assertWithMessage("all refs NOT visible")
+        .that(u.allRefsAreVisible(Collections.emptySet()))
+        .isFalse();
+  }
+
   private void assertNotOwner(String ref, ProjectControl u) {
     assertWithMessage("NOT OWN " + ref).that(u.controlForRef(ref).isOwner()).isFalse();
   }
@@ -181,6 +195,7 @@ public class RefControlTest {
   private final Project.NameKey parentKey = Project.nameKey("parent");
 
   @Inject private AllProjectsName allProjectsName;
+  @Inject private AllUsersName allUsersName;
   @Inject private InMemoryRepositoryManager repoManager;
   @Inject private MetaDataUpdate.Server metaDataUpdateFactory;
   @Inject private ProjectCache projectCache;
@@ -271,6 +286,32 @@ public class RefControlTest {
     assertAdminsAreOwnersAndDevsAreNot();
   }
 
+  @Test
+  public void allRefsAreVisibleForRegularProject() throws Exception {
+    projectOperations
+        .project(localKey)
+        .forUpdate()
+        .add(allow(READ).ref("refs/*").group(DEVS))
+        .add(allow(READ).ref("refs/groups/*").group(DEVS))
+        .add(allow(READ).ref("refs/users/default").group(DEVS))
+        .update();
+
+    assertAllRefsAreVisible(user(localKey, DEVS));
+  }
+
+  @Test
+  public void allRefsAreNotVisibleForAllUsers() throws Exception {
+    projectOperations
+        .project(allUsersName)
+        .forUpdate()
+        .add(allow(READ).ref("refs/*").group(DEVS))
+        .add(allow(READ).ref("refs/groups/*").group(DEVS))
+        .add(allow(READ).ref("refs/users/default").group(DEVS))
+        .update();
+
+    assertAllRefsAreNotVisible(user(allUsersName, DEVS));
+  }
+
   @Test
   public void branchDelegation1() throws Exception {
     projectOperations

lib/jackson/BUILD

@@ -1,5 +1,13 @@
 load("@rules_java//java:defs.bzl", "java_library")
 
+java_library(
+    name = "jackson-annotations",
+    testonly = True,
+    data = ["//lib:LICENSE-Apache2.0"],
+    visibility = ["//visibility:public"],
+    exports = ["@jackson-annotations//jar"],
+)
+
 java_library(
     name = "jackson-core",
     data = ["//lib:LICENSE-Apache2.0"],

lib/nongoogle_test.sh

@@ -12,6 +12,8 @@ grep 'name = "[^"]*"' ${bzl} | sed 's|^[^"]*"||g;s|".*$||g' | sort > $TMP/names
 
 cat << EOF > $TMP/want
 cglib-3_2
+docker-java-api
+docker-java-transport
 dropwizard-core
 duct-tape
 eddsa
@@ -22,6 +24,7 @@ flogger-system-backend
 httpasyncclient
 httpcore-nio
 j2objc
+jackson-annotations
 jackson-core
 jna
 jruby

lib/testcontainers/BUILD

@@ -1,5 +1,21 @@
 load("@rules_java//java:defs.bzl", "java_library")
 
+java_library(
+    name = "docker-java-api",
+    testonly = True,
+    data = ["//lib:LICENSE-Apache2.0"],
+    visibility = ["//visibility:public"],
+    exports = ["@docker-java-api//jar"],
+)
+
+java_library(
+    name = "docker-java-transport",
+    testonly = True,
+    data = ["//lib:LICENSE-Apache2.0"],
+    visibility = ["//visibility:public"],
+    exports = ["@docker-java-transport//jar"],
+)
+
 java_library(
     name = "duct-tape",
     testonly = True,

tools/nongoogle.bzl

@@ -143,18 +143,40 @@ def declare_nongoogle_deps():
         sha1 = "dc13ae4faca6df981fc7aeb5a522d9db446d5d50",
     )
 
-    TESTCONTAINERS_VERSION = "1.14.3"
+    DOCKER_JAVA_VERS = "3.2.5"
+
+    maven_jar(
+        name = "docker-java-api",
+        artifact = "com.github.docker-java:docker-java-api:" + DOCKER_JAVA_VERS,
+        sha1 = "8fe5c5e39f940ce58620e77cedc0a2a52d76f9d8",
+    )
+
+    maven_jar(
+        name = "docker-java-transport",
+        artifact = "com.github.docker-java:docker-java-transport:" + DOCKER_JAVA_VERS,
+        sha1 = "27af0ee7ebc2f5672e23ea64769497b5d55ce3ac",
+    )
+
+    # https://github.com/docker-java/docker-java/blob/3.2.5/pom.xml#L61
+    # <=> DOCKER_JAVA_VERS
+    maven_jar(
+        name = "jackson-annotations",
+        artifact = "com.fasterxml.jackson.core:jackson-annotations:2.10.3",
+        sha1 = "0f63b3b1da563767d04d2e4d3fc1ae0cdeffebe7",
+    )
+
+    TESTCONTAINERS_VERSION = "1.15.0"
 
     maven_jar(
         name = "testcontainers",
         artifact = "org.testcontainers:testcontainers:" + TESTCONTAINERS_VERSION,
-        sha1 = "071fc82ba663f469447a19434e7db90f3a872753",
+        sha1 = "b627535b444d88e7b14953bb953d80d9b7b3bd76",
     )
 
     maven_jar(
         name = "testcontainers-elasticsearch",
         artifact = "org.testcontainers:elasticsearch:" + TESTCONTAINERS_VERSION,
-        sha1 = "3709e2ebb0b6aa4e2ba2b6ca92ffdd3bf637a86c",
+        sha1 = "2bd79fd915e5c7bcf9b5d86cd8e0b7a0fff4b8ce",
     )
 
     maven_jar(