Enable rolling upgrade to next versions

Gerrit version N and N+1 (e.g. Gerrit v3.1 is N, v3.2 is N+1)
have typically a semi-compatible schema which enables the ability
to perform a live-upgrade to the new release without the need of
a general outage.

Document how to enable rolling upgrade mode and make the schema
version check more flexible so that two versions can
share the same repositories over NFS.

Test plan:

1. Install Gerrit version N on two nodes sharing the repositories
   over NFS.
2. Set gerrit.experimentalRollingUpgrade to true
3. Upgrade one of the two nodes to version N+1
4. Both nodes should be able to stop/start without issues
5. Both nodes should be able to serve read/writes without issues
6. Upgrade the other node then confirm as above.

Change-Id: I8ceb352365da2eb553d82495c9635eb034e57352

Documentation/config-gerrit.txt

@@ -2127,6 +2127,36 @@ A good name should be short but precise enough so that users can identify the in
 +
 Defaults to the full hostname of the Gerrit server.
 
+[[gerrit.experimentalRollingUpgrade]]gerrit.experimentalRollingUpgrade::
++
+Enable Gerrit rolling upgrade to the next version.
+For example if Gerrit v3.1 is version N (All-Projects:refs/meta/version=181)
+then its next version N+1 is v3.2 (All-Projects:refs/meta/version=183).
+Allow Gerrit to start even if the underlying schema version has been bumped to
+the next Gerrit version.
++
+Set to true if Gerrit is installed in
+[high-availability configuration](https://gerrit.googlesource.com/plugins/high-availability/+/refs/heads/master/README.md)
+during the rolling upgrade to the next version.
++
+By default false.
++
+The rolling upgrade process, at high level, assumes that Gerrit is installed
+on two or more nodes sharing the repositories over NFS. The upgrade is composed
+of the following steps:
++
+1. Set gerrit.experimentalRollingUpgrade to true on all Gerrit masters
+2. Set the first master unhealthy
+3. Shutdown the first master and [upgrade](install.html#init) to the next version
+4. Startup the first master, wait for the online reindex to complete
+5. Verify the the first master upgrade is successful and online reindex is complete
+6. Set the first master healthy
+7. Repeat steps 2. to 6. for all the other Gerrit nodes
++
+[WARNING]
+Rolling upgrade may or may not be possible depending on the changes introduced
+by the target version of the upgrade. Refer to the release notes and check whether
+the rolling upgrade is possible or not and the associated constraints.
 
 [[gerrit.serverId]]gerrit.serverId::
 +

java/com/google/gerrit/server/schema/NoteDbSchemaVersionCheck.java

@@ -14,15 +14,20 @@
 
 package com.google.gerrit.server.schema;
 
+import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.exceptions.StorageException;
 import com.google.gerrit.extensions.events.LifecycleListener;
 import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.SitePaths;
 import com.google.inject.Inject;
 import com.google.inject.Module;
 import com.google.inject.ProvisionException;
+import org.eclipse.jgit.lib.Config;
 
 public class NoteDbSchemaVersionCheck implements LifecycleListener {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   public static Module module() {
     return new LifecycleModule() {
       @Override
@@ -34,11 +39,16 @@ public class NoteDbSchemaVersionCheck implements LifecycleListener {
 
   private final NoteDbSchemaVersionManager versionManager;
   private final SitePaths sitePaths;
+  private Config gerritConfig;
 
   @Inject
-  NoteDbSchemaVersionCheck(NoteDbSchemaVersionManager versionManager, SitePaths sitePaths) {
+  NoteDbSchemaVersionCheck(
+      NoteDbSchemaVersionManager versionManager,
+      SitePaths sitePaths,
+      @GerritServerConfig Config gerritConfig) {
     this.versionManager = versionManager;
     this.sitePaths = sitePaths;
+    this.gerritConfig = gerritConfig;
   }
 
   @Override
@@ -53,7 +63,18 @@ public class NoteDbSchemaVersionCheck implements LifecycleListener {
                 sitePaths.site_path.toAbsolutePath()));
       }
       int expected = NoteDbSchemaVersions.LATEST;
-      if (current != expected) {
+
+      if (current > expected
+          && gerritConfig.getBoolean("gerrit", "experimentalRollingUpgrade", false)) {
+        logger.atWarning().log(
+            "Gerrit has detected refs/meta/version %d different than the expected %d."
+                + "Bear in mind that this is supported ONLY for rolling upgrades to immediate next "
+                + "Gerrit version (e.g. v3.1 to v3.2). If this is not expected, remove gerrit.experimentalRollingUpgrade "
+                + "from $GERRIT_SITE/etc/gerrit.config and restart Gerrit."
+                + "Please note that gerrit.experimentalRollingUpgrade is intended to be used "
+                + "for the rolling upgrade phase only and should be disabled afterwards.",
+            current, expected);
+      } else if (current != expected) {
         String advice =
             current > expected
                 ? "Downgrade is not supported"

javatests/com/google/gerrit/server/schema/NoteDbSchemaVersionCheckTest.java

@@ -0,0 +1,79 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.schema;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.gerrit.testing.GerritJUnit.assertThrows;
+
+import com.google.gerrit.server.config.AllProjectsName;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.gerrit.testing.InMemoryRepositoryManager;
+import com.google.inject.ProvisionException;
+import java.io.IOException;
+import java.nio.file.Paths;
+import org.eclipse.jgit.lib.Config;
+import org.junit.Before;
+import org.junit.Test;
+
+public class NoteDbSchemaVersionCheckTest {
+  private NoteDbSchemaVersionManager versionManager;
+  private SitePaths sitePaths;
+
+  @Before
+  public void setup() throws Exception {
+    AllProjectsName allProjectsName = new AllProjectsName("All-Projects");
+    GitRepositoryManager repoManager = new InMemoryRepositoryManager();
+    repoManager.createRepository(allProjectsName);
+    versionManager = new NoteDbSchemaVersionManager(allProjectsName, repoManager);
+    versionManager.init();
+
+    sitePaths = new SitePaths(Paths.get("/tmp/foo"));
+  }
+
+  @Test
+  public void shouldNotFailIfCurrentVersionIsExpected() {
+    new NoteDbSchemaVersionCheck(versionManager, sitePaths, new Config()).start();
+    // No exceptions should be thrown
+  }
+
+  @Test
+  public void shouldFailIfCurrentVersionIsOneMoreThanExpected() throws IOException {
+    versionManager.increment(NoteDbSchemaVersions.LATEST);
+
+    ProvisionException e =
+        assertThrows(
+            ProvisionException.class,
+            () -> new NoteDbSchemaVersionCheck(versionManager, sitePaths, new Config()).start());
+
+    assertThat(e)
+        .hasMessageThat()
+        .contains("Unsupported schema version " + (NoteDbSchemaVersions.LATEST + 1));
+  }
+
+  @Test
+  public void
+      shouldNotFailWithExperimentalRollingUpgradeEnabledAndCurrentVersionIsOneMoreThanExpected()
+          throws IOException {
+    Config gerritConfig = new Config();
+    gerritConfig.setBoolean("gerrit", null, "experimentalRollingUpgrade", true);
+    versionManager.increment(NoteDbSchemaVersions.LATEST);
+
+    NoteDbSchemaVersionCheck versionCheck =
+        new NoteDbSchemaVersionCheck(versionManager, sitePaths, gerritConfig);
+    versionCheck.start();
+    // No exceptions should be thrown
+  }
+}