NoteDb schema upgrades

Analogous to the schema version number in the CurrentSchemaVersion
table, store the current schema version in NoteDb as an int blob in
refs/meta/version in All-Projects. Version numbers start at 180, which
continues the numbering and class naming scheme from ReviewDb, plus a
gap to allow for a last few schema upgrades in the stable-2.16 branch.

Although NoteDb gives us the flexibility to do more interesting things
in terms of out-of-order or optional schema upgrades, this
implementation sticks to the old ReviewDb way of doing things with a
single monotonically increasing version.

The implementation is roughly similar to the ReviewDb implementation,
where we have NoteDbSchemaVersion{Check,Updater} classes to enforce that
the running server has been fully upgraded, and a simple loop run during
init to upgrade the schema. To make NoteDb migration tests pass with
GERRIT_NOTEDB=OFF, these only take effect when NoteDb is enabled at
setup time. In reality, this hack is short-lived, since we will be
removing ReviewDb support entirely quite soon.

The main difference in implementation is that we no longer construct
NoteDbSchemaVersions using Guice, and instead require a certain
constructor signature for implementations. Not constructing these via
Guice also allows us to do away with the chained
Provider<Schema_$PREVIOUS> constructor arguments. These historically
caused Guice performance problems, which is why they got converted to
Providers in the first place. Rather than preserve that hacky logic,
just don't use Guice. This is not such a painful amount of reflection.

For now, leave the ReviewDb schema upgrade code around, so technically
we can upgrade both ReviewDb and NoteDb schemas in a single binary. In
practice, however, a followup change should be able to completely delete
all old schema upgrade code.

No additional work is done in this series in order to accommodate any
last-minute schema upgrades on the stable-2.16 branch. In order to allow
admins to upgrade directly from 2.16, even if there are new upgrades in
2.16.1 or later, we will have to add idempotent NoteDb schema upgrade
implementations of those upgrades. This is a small amount of extra work,
but there should not be many of these. It also does not require any
additional support in this change.

Change-Id: Ibd2868b8de8de023c8f2c661e2ce3a2b21f3a2f5

java/com/google/gerrit/server/schema/AllProjectsCreator.java

@@ -48,9 +48,11 @@ import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
 import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.git.meta.MetaDataUpdate;
 import com.google.gerrit.server.group.SystemGroupBackend;
+import com.google.gerrit.server.notedb.NoteDbSchemaVersionManager;
 import com.google.gerrit.server.notedb.NotesMigration;
 import com.google.gerrit.server.notedb.RepoSequence;
 import com.google.gerrit.server.project.ProjectConfig;
+import com.google.gwtorm.server.OrmException;
 import com.google.inject.Inject;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -73,6 +75,7 @@ public class AllProjectsCreator {
   private final AllProjectsName allProjectsName;
   private final PersonIdent serverUser;
   private final NotesMigration notesMigration;
+  private final NoteDbSchemaVersionManager versionManager;
   private final ProjectConfig.Factory projectConfigFactory;
   private final GroupReference anonymous;
   private final GroupReference registered;
@@ -91,12 +94,14 @@ public class AllProjectsCreator {
       AllProjectsName allProjectsName,
       @GerritPersonIdent PersonIdent serverUser,
       NotesMigration notesMigration,
+      NoteDbSchemaVersionManager versionManager,
       SystemGroupBackend systemGroupBackend,
       ProjectConfig.Factory projectConfigFactory) {
     this.repositoryManager = repositoryManager;
     this.allProjectsName = allProjectsName;
     this.serverUser = serverUser;
     this.notesMigration = notesMigration;
+    this.versionManager = versionManager;
     this.projectConfigFactory = projectConfigFactory;
 
     this.anonymous = systemGroupBackend.getGroup(ANONYMOUS_USERS);
@@ -145,7 +150,7 @@ public class AllProjectsCreator {
     return this;
   }
 
-  public void create() throws IOException, ConfigInvalidException {
+  public void create() throws IOException, ConfigInvalidException, OrmException {
     try (Repository git = repositoryManager.openRepository(allProjectsName)) {
       initAllProjects(git);
     } catch (RepositoryNotFoundException notFound) {
@@ -162,7 +167,8 @@ public class AllProjectsCreator {
     }
   }
 
-  private void initAllProjects(Repository git) throws IOException, ConfigInvalidException {
+  private void initAllProjects(Repository git)
+      throws IOException, ConfigInvalidException, OrmException {
     BatchRefUpdate bru = git.getRefDatabase().newBatchUpdate();
     try (MetaDataUpdate md =
         new MetaDataUpdate(GitReferenceUpdated.DISABLED, allProjectsName, git, bru)) {
@@ -231,6 +237,7 @@ public class AllProjectsCreator {
 
       config.commitToNewRef(md, RefNames.REFS_CONFIG);
       initSequences(git, bru);
+      initSchemaVersion();
       execute(git, bru);
     }
   }
@@ -268,6 +275,12 @@ public class AllProjectsCreator {
     }
   }
 
+  private void initSchemaVersion() throws IOException, OrmException {
+    if (notesMigration.commitChangeWrites()) {
+      versionManager.init();
+    }
+  }
+
   private void execute(Repository git, BatchRefUpdate bru) throws IOException {
     try (RevWalk rw = new RevWalk(git)) {
       bru.execute(rw, NullProgressMonitor.INSTANCE);

java/com/google/gerrit/server/schema/NoteDbSchemaUpdater.java

@@ -0,0 +1,102 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.schema;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.gerrit.server.notedb.NoteDbSchemaVersionManager;
+import com.google.gerrit.server.notedb.NotesMigration;
+import com.google.gwtorm.server.OrmException;
+import com.google.inject.Inject;
+import java.util.stream.IntStream;
+
+public class NoteDbSchemaUpdater {
+  private final NotesMigration notesMigration;
+  private final NoteDbSchemaVersionManager versionManager;
+  private final NoteDbSchemaVersion.Arguments args;
+  private final ImmutableSortedMap<Integer, Class<? extends NoteDbSchemaVersion>> schemaVersions;
+
+  @Inject
+  NoteDbSchemaUpdater(
+      NotesMigration notesMigration,
+      NoteDbSchemaVersionManager versionManager,
+      NoteDbSchemaVersion.Arguments args) {
+    this(notesMigration, versionManager, args, NoteDbSchemaVersions.ALL);
+  }
+
+  NoteDbSchemaUpdater(
+      NotesMigration notesMigration,
+      NoteDbSchemaVersionManager versionManager,
+      NoteDbSchemaVersion.Arguments args,
+      ImmutableSortedMap<Integer, Class<? extends NoteDbSchemaVersion>> schemaVersions) {
+    this.notesMigration = notesMigration;
+    this.versionManager = versionManager;
+    this.args = args;
+    this.schemaVersions = schemaVersions;
+  }
+
+  public void update(UpdateUI ui) throws OrmException {
+    if (!notesMigration.commitChangeWrites()) {
+      // TODO(dborowitz): Only necessary to make migration tests pass; remove when NoteDb is the
+      // only option.
+      return;
+    }
+    for (int nextVersion : requiredUpgrades(versionManager.read(), schemaVersions.keySet())) {
+      try {
+        ui.message(String.format("Migrating data to schema %d ...", nextVersion));
+        NoteDbSchemaVersions.get(schemaVersions, nextVersion, args).upgrade(ui);
+        versionManager.increment(nextVersion - 1);
+      } catch (Exception e) {
+        throw new OrmException(
+            String.format("Failed to upgrade to schema version %d", nextVersion), e);
+      }
+    }
+  }
+
+  @VisibleForTesting
+  static ImmutableList<Integer> requiredUpgrades(
+      int currentVersion, ImmutableSortedSet<Integer> allVersions) throws OrmException {
+    int firstVersion = allVersions.first();
+    int latestVersion = allVersions.last();
+    if (currentVersion == latestVersion) {
+      return ImmutableList.of();
+    } else if (currentVersion > latestVersion) {
+      throw new OrmException(
+          String.format(
+              "Cannot downgrade NoteDb schema from version %d to %d",
+              currentVersion, latestVersion));
+    }
+
+    int firstUpgradeVersion;
+    if (currentVersion == 0) {
+      // Bootstrap NoteDb version to minimum supported schema number.
+      firstUpgradeVersion = firstVersion;
+    } else {
+      if (currentVersion < firstVersion - 1) {
+        throw new OrmException(
+            String.format(
+                "Cannot skip NoteDb schema from version %d to %d", currentVersion, firstVersion));
+      }
+      firstUpgradeVersion = currentVersion + 1;
+    }
+    return IntStream.rangeClosed(firstUpgradeVersion, latestVersion)
+        .boxed()
+        .collect(toImmutableList());
+  }
+}

java/com/google/gerrit/server/schema/NoteDbSchemaVersion.java

@@ -0,0 +1,43 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.schema;
+
+import com.google.gerrit.server.config.AllProjectsName;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+/**
+ * Schema upgrade implementation.
+ *
+ * <p>Implementations must define a single public constructor that takes an {@link Arguments}. The
+ * recommended idiom is to pull out whichever individual fields from the {@code Arguments} are
+ * required by this implementation.
+ */
+interface NoteDbSchemaVersion {
+  @Singleton
+  class Arguments {
+    final GitRepositoryManager repoManager;
+    final AllProjectsName allProjects;
+
+    @Inject
+    Arguments(GitRepositoryManager repoManager, AllProjectsName allProjects) {
+      this.repoManager = repoManager;
+      this.allProjects = allProjects;
+    }
+  }
+
+  void upgrade(UpdateUI ui) throws Exception;
+}

java/com/google/gerrit/server/schema/NoteDbSchemaVersionCheck.java

@@ -0,0 +1,89 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.schema;
+
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.notedb.NoteDbSchemaVersionManager;
+import com.google.gerrit.server.notedb.NotesMigration;
+import com.google.gwtorm.server.OrmException;
+import com.google.inject.Inject;
+import com.google.inject.Module;
+import com.google.inject.ProvisionException;
+
+public class NoteDbSchemaVersionCheck implements LifecycleListener {
+  public static Module module() {
+    return new LifecycleModule() {
+      @Override
+      protected void configure() {
+        listener().to(NoteDbSchemaVersionCheck.class);
+      }
+    };
+  }
+
+  private final NotesMigration notesMigration;
+  private final NoteDbSchemaVersionManager versionManager;
+  private final SitePaths sitePaths;
+
+  @Inject
+  NoteDbSchemaVersionCheck(
+      NotesMigration notesMigration,
+      NoteDbSchemaVersionManager versionManager,
+      SitePaths sitePaths) {
+    this.notesMigration = notesMigration;
+    this.versionManager = versionManager;
+    this.sitePaths = sitePaths;
+  }
+
+  @Override
+  public void start() {
+    if (!notesMigration.commitChangeWrites()) {
+      // TODO(dborowitz): Only necessary to make migration tests pass; remove when NoteDb is the
+      // only option.
+      return;
+    }
+    try {
+      int current = versionManager.read();
+      if (current == 0) {
+        throw new ProvisionException(
+            String.format(
+                "Schema not yet initialized. Run init to initialize the schema:\n"
+                    + "$ java -jar gerrit.war init -d %s",
+                sitePaths.site_path.toAbsolutePath()));
+      }
+      int expected = NoteDbSchemaVersions.LATEST;
+      if (current != expected) {
+        String advice =
+            current > expected
+                ? "Downgrade is not supported"
+                : String.format(
+                    "Run init to upgrade:\n$ java -jar %s init -d %s",
+                    sitePaths.gerrit_war.toAbsolutePath(), sitePaths.site_path.toAbsolutePath());
+        throw new ProvisionException(
+            String.format(
+                "Unsupported schema version %d; expected schema version %d. %s",
+                current, expected, advice));
+      }
+    } catch (OrmException e) {
+      throw new ProvisionException("Failed to read NoteDb schema version", e);
+    }
+  }
+
+  @Override
+  public void stop() {
+    // Do nothing.
+  }
+}

java/com/google/gerrit/server/schema/NoteDbSchemaVersions.java

@@ -0,0 +1,62 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.schema;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap;
+import static java.util.Comparator.naturalOrder;
+
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.primitives.Ints;
+import com.google.gerrit.server.UsedAt;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+public class NoteDbSchemaVersions {
+  static final ImmutableSortedMap<Integer, Class<? extends NoteDbSchemaVersion>> ALL =
+      // List all supported NoteDb schema versions here.
+      Stream.of(Schema_180.class)
+          .collect(toImmutableSortedMap(naturalOrder(), v -> guessVersion(v).get(), v -> v));
+
+  public static final int FIRST = ALL.firstKey();
+  public static final int LATEST = ALL.lastKey();
+
+  // TODO(dborowitz): Migrate delete-project plugin to use this implementation.
+  @UsedAt(UsedAt.Project.PLUGIN_DELETE_PROJECT)
+  public static Optional<Integer> guessVersion(Class<?> c) {
+    String prefix = "Schema_";
+    if (!c.getSimpleName().startsWith(prefix)) {
+      return Optional.empty();
+    }
+    return Optional.ofNullable(Ints.tryParse(c.getSimpleName().substring(prefix.length())));
+  }
+
+  public static NoteDbSchemaVersion get(
+      ImmutableSortedMap<Integer, Class<? extends NoteDbSchemaVersion>> schemaVersions,
+      int i,
+      NoteDbSchemaVersion.Arguments args) {
+    Class<? extends NoteDbSchemaVersion> clazz = schemaVersions.get(i);
+    checkArgument(clazz != null, "Schema version not found: %s", i);
+    try {
+      return clazz.getDeclaredConstructor(NoteDbSchemaVersion.Arguments.class).newInstance(args);
+    } catch (InstantiationException
+        | IllegalAccessException
+        | NoSuchMethodException
+        | InvocationTargetException e) {
+      throw new IllegalStateException("failed to invoke constructor on " + clazz.getName(), e);
+    }
+  }
+}

java/com/google/gerrit/server/schema/ReviewDbSchemaVersion.java

@@ -14,6 +14,8 @@
 
 package com.google.gerrit.server.schema;
 
+import static com.google.common.base.Preconditions.checkState;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.Lists;
@@ -36,8 +38,17 @@ import java.util.concurrent.TimeUnit;
 /** A version of the database schema. */
 public abstract class ReviewDbSchemaVersion {
   /** The current schema version. */
+  // DO NOT upgrade this version in the master branch. Future versions must all be implemented as
+  // NoteDbSchemaVersions. It may be upgraded on the stable-2.16 branch, in which case this will
+  // need to be updated upon merging. In any case, this number must not exceed the first NoteDb
+  // schema version (180).
   public static final Class<Schema_170> C = Schema_170.class;
 
+  static {
+    checkState(C.equals(Schema_170.class));
+    checkState(guessVersion(C) < 180);
+  }
+
   public static int getBinaryVersion() {
     return guessVersion(C);
   }

java/com/google/gerrit/server/schema/Schema_180.java

@@ -0,0 +1,27 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.schema;
+
+public class Schema_180 implements NoteDbSchemaVersion {
+  @SuppressWarnings("unused")
+  Schema_180(Arguments args) {
+    // Do nothing.
+  }
+
+  @Override
+  public void upgrade(UpdateUI ui) {
+    // Do nothing; only used to populate the version ref, which is done by the caller.
+  }
+}