NoteDb schema upgrades

Analogous to the schema version number in the CurrentSchemaVersion
table, store the current schema version in NoteDb as an int blob in
refs/meta/version in All-Projects. Version numbers start at 180, which
continues the numbering and class naming scheme from ReviewDb, plus a
gap to allow for a last few schema upgrades in the stable-2.16 branch.

Although NoteDb gives us the flexibility to do more interesting things
in terms of out-of-order or optional schema upgrades, this
implementation sticks to the old ReviewDb way of doing things with a
single monotonically increasing version.

The implementation is roughly similar to the ReviewDb implementation,
where we have NoteDbSchemaVersion{Check,Updater} classes to enforce that
the running server has been fully upgraded, and a simple loop run during
init to upgrade the schema. To make NoteDb migration tests pass with
GERRIT_NOTEDB=OFF, these only take effect when NoteDb is enabled at
setup time. In reality, this hack is short-lived, since we will be
removing ReviewDb support entirely quite soon.

The main difference in implementation is that we no longer construct
NoteDbSchemaVersions using Guice, and instead require a certain
constructor signature for implementations. Not constructing these via
Guice also allows us to do away with the chained
Provider<Schema_$PREVIOUS> constructor arguments. These historically
caused Guice performance problems, which is why they got converted to
Providers in the first place. Rather than preserve that hacky logic,
just don't use Guice. This is not such a painful amount of reflection.

For now, leave the ReviewDb schema upgrade code around, so technically
we can upgrade both ReviewDb and NoteDb schemas in a single binary. In
practice, however, a followup change should be able to completely delete
all old schema upgrade code.

No additional work is done in this series in order to accommodate any
last-minute schema upgrades on the stable-2.16 branch. In order to allow
admins to upgrade directly from 2.16, even if there are new upgrades in
2.16.1 or later, we will have to add idempotent NoteDb schema upgrade
implementations of those upgrades. This is a small amount of extra work,
but there should not be many of these. It also does not require any
additional support in this change.

Change-Id: Ibd2868b8de8de023c8f2c661e2ce3a2b21f3a2f5
This commit is contained in:
Dave Borowitz
2018-11-13 16:40:25 -08:00
parent 2a84138f06
commit 9db5a7462d
17 changed files with 882 additions and 45 deletions

View File

@@ -90,6 +90,7 @@ import com.google.gerrit.server.schema.DataSourceProvider;
import com.google.gerrit.server.schema.DataSourceType;
import com.google.gerrit.server.schema.DatabaseModule;
import com.google.gerrit.server.schema.JdbcAccountPatchReviewStore;
import com.google.gerrit.server.schema.NoteDbSchemaVersionCheck;
import com.google.gerrit.server.schema.ReviewDbSchemaModule;
import com.google.gerrit.server.schema.ReviewDbSchemaVersionCheck;
import com.google.gerrit.server.securestore.SecureStoreClassName;
@@ -311,6 +312,7 @@ public class WebAppInitializer extends GuiceServletContextListener implements Fi
final List<Module> modules = new ArrayList<>();
modules.add(new ReviewDbSchemaModule());
modules.add(ReviewDbSchemaVersionCheck.module());
modules.add(NoteDbSchemaVersionCheck.module());
modules.add(new AuthConfigModule());
return dbInjector.createChildInjector(modules);
}

View File

@@ -97,6 +97,7 @@ import com.google.gerrit.server.restapi.RestApiModule;
import com.google.gerrit.server.schema.DataSourceProvider;
import com.google.gerrit.server.schema.InMemoryAccountPatchReviewStore;
import com.google.gerrit.server.schema.JdbcAccountPatchReviewStore;
import com.google.gerrit.server.schema.NoteDbSchemaVersionCheck;
import com.google.gerrit.server.schema.ReviewDbSchemaVersionCheck;
import com.google.gerrit.server.securestore.DefaultSecureStore;
import com.google.gerrit.server.securestore.SecureStore;
@@ -398,6 +399,7 @@ public class Daemon extends SiteProgram {
private Injector createSysInjector() {
final List<Module> modules = new ArrayList<>();
modules.add(ReviewDbSchemaVersionCheck.module());
modules.add(NoteDbSchemaVersionCheck.module());
modules.add(new DropWizardMetricMaker.RestModule());
modules.add(new LogFileCompressor.Module());

View File

@@ -29,6 +29,7 @@ import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.meta.MetaDataUpdate;
import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
import com.google.gerrit.server.schema.NoteDbSchemaVersionCheck;
import com.google.gerrit.server.schema.ReviewDbSchemaVersionCheck;
import com.google.gwtorm.server.OrmDuplicateKeyException;
import com.google.inject.Inject;
@@ -56,6 +57,7 @@ public class LocalUsernamesToLowerCase extends SiteProgram {
public int run() throws Exception {
Injector dbInjector = createDbInjector(MULTI_USER);
manager.add(dbInjector, dbInjector.createChildInjector(ReviewDbSchemaVersionCheck.module()));
manager.add(dbInjector, dbInjector.createChildInjector(NoteDbSchemaVersionCheck.module()));
manager.start();
dbInjector
.createChildInjector(

View File

@@ -42,6 +42,7 @@ import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.plugins.JarScanner;
import com.google.gerrit.server.schema.NoteDbSchemaUpdater;
import com.google.gerrit.server.schema.ReviewDbFactory;
import com.google.gerrit.server.schema.ReviewDbSchemaUpdater;
import com.google.gerrit.server.schema.UpdateUI;
@@ -358,7 +359,8 @@ public class BaseInit extends SiteProgram {
public final ConsoleUI ui;
public final SitePaths site;
public final InitFlags flags;
final ReviewDbSchemaUpdater schemaUpdater;
final ReviewDbSchemaUpdater reviewDbSchemaUpdater;
final NoteDbSchemaUpdater noteDbSchemaUpdater;
final SchemaFactory<ReviewDb> schema;
final GitRepositoryManager repositoryManager;
@@ -367,57 +369,23 @@ public class BaseInit extends SiteProgram {
ConsoleUI ui,
SitePaths site,
InitFlags flags,
ReviewDbSchemaUpdater schemaUpdater,
ReviewDbSchemaUpdater reviewDbSchemaUpdater,
NoteDbSchemaUpdater noteDbSchemaUpdater,
@ReviewDbFactory SchemaFactory<ReviewDb> schema,
GitRepositoryManager repositoryManager) {
this.ui = ui;
this.site = site;
this.flags = flags;
this.schemaUpdater = schemaUpdater;
this.reviewDbSchemaUpdater = reviewDbSchemaUpdater;
this.noteDbSchemaUpdater = noteDbSchemaUpdater;
this.schema = schema;
this.repositoryManager = repositoryManager;
}
void upgradeSchema() throws OrmException {
final List<String> pruneList = new ArrayList<>();
schemaUpdater.update(
new UpdateUI() {
@Override
public void message(String message) {
System.err.println(message);
System.err.flush();
}
@Override
public boolean yesno(boolean defaultValue, String message) {
return ui.yesno(defaultValue, message);
}
@Override
public void waitForUser() {
ui.waitForUser();
}
@Override
public String readString(
String defaultValue, Set<String> allowedValues, String message) {
return ui.readString(defaultValue, allowedValues, message);
}
@Override
public boolean isBatch() {
return ui.isBatch();
}
@Override
public void pruneSchema(StatementExecutor e, List<String> prune) {
for (String p : prune) {
if (!pruneList.contains(p)) {
pruneList.add(p);
}
}
}
});
UpdateUI uiImpl = new UpdateUIImpl(ui, pruneList);
reviewDbSchemaUpdater.update(uiImpl);
if (!pruneList.isEmpty()) {
StringBuilder msg = new StringBuilder();
@@ -442,6 +410,53 @@ public class BaseInit extends SiteProgram {
}
}
}
noteDbSchemaUpdater.update(uiImpl);
}
private static class UpdateUIImpl implements UpdateUI {
private final ConsoleUI consoleUi;
private final List<String> pruneList;
UpdateUIImpl(ConsoleUI consoleUi, List<String> pruneList) {
this.consoleUi = consoleUi;
this.pruneList = pruneList;
}
@Override
public void message(String message) {
System.err.println(message);
System.err.flush();
}
@Override
public boolean yesno(boolean defaultValue, String message) {
return consoleUi.yesno(defaultValue, message);
}
@Override
public void waitForUser() {
consoleUi.waitForUser();
}
@Override
public String readString(String defaultValue, Set<String> allowedValues, String message) {
return consoleUi.readString(defaultValue, allowedValues, message);
}
@Override
public boolean isBatch() {
return consoleUi.isBatch();
}
@Override
public void pruneSchema(StatementExecutor e, List<String> prune) {
for (String p : prune) {
if (!pruneList.contains(p)) {
pruneList.add(p);
}
}
}
}
}

View File

@@ -49,6 +49,9 @@ public class RefNames {
/** Sequence counters in NoteDb. */
public static final String REFS_SEQUENCES = "refs/sequences/";
/** NoteDb schema version number. */
public static final String REFS_VERSION = "refs/meta/version";
/**
* Prefix applied to merge commit base nodes.
*

View File

@@ -0,0 +1,101 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb;
import static com.google.gerrit.reviewdb.client.RefNames.REFS_VERSION;
import com.google.common.annotations.VisibleForTesting;
import com.google.gerrit.server.config.AllProjectsName;
import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gwtorm.server.OrmException;
import java.io.IOException;
import java.util.Optional;
import javax.inject.Inject;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevWalk;
public class NoteDbSchemaVersionManager {
private final AllProjectsName allProjectsName;
private final GitRepositoryManager repoManager;
@Inject
@VisibleForTesting
public NoteDbSchemaVersionManager(
AllProjectsName allProjectsName, GitRepositoryManager repoManager) {
// Can't inject GitReferenceUpdated here because it has dependencies that are not always
// available in this injector (e.g. during init). This is ok for now since no other ref updates
// during init are available to plugins, and there are not any other use cases for listening for
// updates to the version ref.
this.allProjectsName = allProjectsName;
this.repoManager = repoManager;
}
public int read() throws OrmException {
try (Repository repo = repoManager.openRepository(allProjectsName)) {
return IntBlob.parse(repo, REFS_VERSION).map(IntBlob::value).orElse(0);
} catch (IOException e) {
throw new OrmException("Failed to read " + REFS_VERSION, e);
}
}
public void init() throws IOException, OrmException {
try (Repository repo = repoManager.openRepository(allProjectsName);
RevWalk rw = new RevWalk(repo)) {
Optional<IntBlob> old = IntBlob.parse(repo, REFS_VERSION, rw);
if (old.isPresent()) {
throw new OrmException(
String.format(
"Expected no old version for %s, found %s", REFS_VERSION, old.get().value()));
}
IntBlob.store(
repo,
rw,
allProjectsName,
REFS_VERSION,
old.map(IntBlob::id).orElse(ObjectId.zeroId()),
// TODO(dborowitz): Find some way to not hard-code this constant here. We can't depend on
// NoteDbSchemaVersions from this package, because the schema java_library depends on the
// server java_library, so that would add a circular dependency. But *this* class must
// live in the server library, because it's used by things like NoteDbMigrator. One
// option: once NoteDbMigrator goes away, this class could move back to the schema
// subpackage.
180,
GitReferenceUpdated.DISABLED);
}
}
public void increment(int expectedOldVersion) throws IOException, OrmException {
try (Repository repo = repoManager.openRepository(allProjectsName);
RevWalk rw = new RevWalk(repo)) {
Optional<IntBlob> old = IntBlob.parse(repo, REFS_VERSION, rw);
if (old.isPresent() && old.get().value() != expectedOldVersion) {
throw new OrmException(
String.format(
"Expected old version %d for %s, found %d",
expectedOldVersion, REFS_VERSION, old.get().value()));
}
IntBlob.store(
repo,
rw,
allProjectsName,
REFS_VERSION,
old.map(IntBlob::id).orElse(ObjectId.zeroId()),
expectedOldVersion + 1,
GitReferenceUpdated.DISABLED);
}
}
}

View File

@@ -64,6 +64,7 @@ import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.notedb.ChangeBundleReader;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.notedb.MutableNotesMigration;
import com.google.gerrit.server.notedb.NoteDbSchemaVersionManager;
import com.google.gerrit.server.notedb.NoteDbTable;
import com.google.gerrit.server.notedb.NoteDbUpdateManager;
import com.google.gerrit.server.notedb.NotesMigrationState;
@@ -151,6 +152,7 @@ public class NoteDbMigrator implements AutoCloseable {
private final MutableNotesMigration globalNotesMigration;
private final PrimaryStorageMigrator primaryStorageMigrator;
private final PluginSetContext<NotesMigrationStateListener> listeners;
private final NoteDbSchemaVersionManager versionManager;
private int threads;
private ImmutableList<Project.NameKey> projects = ImmutableList.of();
@@ -179,7 +181,8 @@ public class NoteDbMigrator implements AutoCloseable {
WorkQueue workQueue,
MutableNotesMigration globalNotesMigration,
PrimaryStorageMigrator primaryStorageMigrator,
PluginSetContext<NotesMigrationStateListener> listeners) {
PluginSetContext<NotesMigrationStateListener> listeners,
NoteDbSchemaVersionManager versionManager) {
// Reload gerrit.config/notedb.config on each migrator invocation, in case a previous
// migration in the same process modified the on-disk contents. This ensures the defaults for
// trial/autoMigrate get set correctly below.
@@ -199,6 +202,7 @@ public class NoteDbMigrator implements AutoCloseable {
this.globalNotesMigration = globalNotesMigration;
this.primaryStorageMigrator = primaryStorageMigrator;
this.listeners = listeners;
this.versionManager = versionManager;
this.trial = getTrialMode(cfg);
this.autoMigrate = getAutoMigrate(cfg);
}
@@ -361,6 +365,7 @@ public class NoteDbMigrator implements AutoCloseable {
globalNotesMigration,
primaryStorageMigrator,
listeners,
versionManager,
threads > 1
? MoreExecutors.listeningDecorator(
workQueue.createQueue(threads, "RebuildChange", true))
@@ -391,6 +396,7 @@ public class NoteDbMigrator implements AutoCloseable {
private final MutableNotesMigration globalNotesMigration;
private final PrimaryStorageMigrator primaryStorageMigrator;
private final PluginSetContext<NotesMigrationStateListener> listeners;
private final NoteDbSchemaVersionManager versionManager;
private final ListeningExecutorService executor;
private final ImmutableList<Project.NameKey> projects;
@@ -417,6 +423,7 @@ public class NoteDbMigrator implements AutoCloseable {
MutableNotesMigration globalNotesMigration,
PrimaryStorageMigrator primaryStorageMigrator,
PluginSetContext<NotesMigrationStateListener> listeners,
NoteDbSchemaVersionManager versionManager,
ListeningExecutorService executor,
ImmutableList<Project.NameKey> projects,
ImmutableList<Change.Id> changes,
@@ -447,6 +454,7 @@ public class NoteDbMigrator implements AutoCloseable {
this.globalNotesMigration = globalNotesMigration;
this.primaryStorageMigrator = primaryStorageMigrator;
this.listeners = listeners;
this.versionManager = versionManager;
this.executor = executor;
this.projects = projects;
this.changes = changes;
@@ -546,7 +554,9 @@ public class NoteDbMigrator implements AutoCloseable {
}
}
private NotesMigrationState turnOnWrites(NotesMigrationState prev) throws IOException {
private NotesMigrationState turnOnWrites(NotesMigrationState prev)
throws OrmException, IOException {
versionManager.init();
return saveState(prev, WRITE);
}

View File

@@ -48,9 +48,11 @@ import com.google.gerrit.server.extensions.events.GitReferenceUpdated;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.meta.MetaDataUpdate;
import com.google.gerrit.server.group.SystemGroupBackend;
import com.google.gerrit.server.notedb.NoteDbSchemaVersionManager;
import com.google.gerrit.server.notedb.NotesMigration;
import com.google.gerrit.server.notedb.RepoSequence;
import com.google.gerrit.server.project.ProjectConfig;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import java.io.IOException;
import java.util.ArrayList;
@@ -73,6 +75,7 @@ public class AllProjectsCreator {
private final AllProjectsName allProjectsName;
private final PersonIdent serverUser;
private final NotesMigration notesMigration;
private final NoteDbSchemaVersionManager versionManager;
private final ProjectConfig.Factory projectConfigFactory;
private final GroupReference anonymous;
private final GroupReference registered;
@@ -91,12 +94,14 @@ public class AllProjectsCreator {
AllProjectsName allProjectsName,
@GerritPersonIdent PersonIdent serverUser,
NotesMigration notesMigration,
NoteDbSchemaVersionManager versionManager,
SystemGroupBackend systemGroupBackend,
ProjectConfig.Factory projectConfigFactory) {
this.repositoryManager = repositoryManager;
this.allProjectsName = allProjectsName;
this.serverUser = serverUser;
this.notesMigration = notesMigration;
this.versionManager = versionManager;
this.projectConfigFactory = projectConfigFactory;
this.anonymous = systemGroupBackend.getGroup(ANONYMOUS_USERS);
@@ -145,7 +150,7 @@ public class AllProjectsCreator {
return this;
}
public void create() throws IOException, ConfigInvalidException {
public void create() throws IOException, ConfigInvalidException, OrmException {
try (Repository git = repositoryManager.openRepository(allProjectsName)) {
initAllProjects(git);
} catch (RepositoryNotFoundException notFound) {
@@ -162,7 +167,8 @@ public class AllProjectsCreator {
}
}
private void initAllProjects(Repository git) throws IOException, ConfigInvalidException {
private void initAllProjects(Repository git)
throws IOException, ConfigInvalidException, OrmException {
BatchRefUpdate bru = git.getRefDatabase().newBatchUpdate();
try (MetaDataUpdate md =
new MetaDataUpdate(GitReferenceUpdated.DISABLED, allProjectsName, git, bru)) {
@@ -231,6 +237,7 @@ public class AllProjectsCreator {
config.commitToNewRef(md, RefNames.REFS_CONFIG);
initSequences(git, bru);
initSchemaVersion();
execute(git, bru);
}
}
@@ -268,6 +275,12 @@ public class AllProjectsCreator {
}
}
private void initSchemaVersion() throws IOException, OrmException {
if (notesMigration.commitChangeWrites()) {
versionManager.init();
}
}
private void execute(Repository git, BatchRefUpdate bru) throws IOException {
try (RevWalk rw = new RevWalk(git)) {
bru.execute(rw, NullProgressMonitor.INSTANCE);

View File

@@ -0,0 +1,102 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.schema;
import static com.google.common.collect.ImmutableList.toImmutableList;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.ImmutableSortedSet;
import com.google.gerrit.server.notedb.NoteDbSchemaVersionManager;
import com.google.gerrit.server.notedb.NotesMigration;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import java.util.stream.IntStream;
public class NoteDbSchemaUpdater {
private final NotesMigration notesMigration;
private final NoteDbSchemaVersionManager versionManager;
private final NoteDbSchemaVersion.Arguments args;
private final ImmutableSortedMap<Integer, Class<? extends NoteDbSchemaVersion>> schemaVersions;
@Inject
NoteDbSchemaUpdater(
NotesMigration notesMigration,
NoteDbSchemaVersionManager versionManager,
NoteDbSchemaVersion.Arguments args) {
this(notesMigration, versionManager, args, NoteDbSchemaVersions.ALL);
}
NoteDbSchemaUpdater(
NotesMigration notesMigration,
NoteDbSchemaVersionManager versionManager,
NoteDbSchemaVersion.Arguments args,
ImmutableSortedMap<Integer, Class<? extends NoteDbSchemaVersion>> schemaVersions) {
this.notesMigration = notesMigration;
this.versionManager = versionManager;
this.args = args;
this.schemaVersions = schemaVersions;
}
public void update(UpdateUI ui) throws OrmException {
if (!notesMigration.commitChangeWrites()) {
// TODO(dborowitz): Only necessary to make migration tests pass; remove when NoteDb is the
// only option.
return;
}
for (int nextVersion : requiredUpgrades(versionManager.read(), schemaVersions.keySet())) {
try {
ui.message(String.format("Migrating data to schema %d ...", nextVersion));
NoteDbSchemaVersions.get(schemaVersions, nextVersion, args).upgrade(ui);
versionManager.increment(nextVersion - 1);
} catch (Exception e) {
throw new OrmException(
String.format("Failed to upgrade to schema version %d", nextVersion), e);
}
}
}
@VisibleForTesting
static ImmutableList<Integer> requiredUpgrades(
int currentVersion, ImmutableSortedSet<Integer> allVersions) throws OrmException {
int firstVersion = allVersions.first();
int latestVersion = allVersions.last();
if (currentVersion == latestVersion) {
return ImmutableList.of();
} else if (currentVersion > latestVersion) {
throw new OrmException(
String.format(
"Cannot downgrade NoteDb schema from version %d to %d",
currentVersion, latestVersion));
}
int firstUpgradeVersion;
if (currentVersion == 0) {
// Bootstrap NoteDb version to minimum supported schema number.
firstUpgradeVersion = firstVersion;
} else {
if (currentVersion < firstVersion - 1) {
throw new OrmException(
String.format(
"Cannot skip NoteDb schema from version %d to %d", currentVersion, firstVersion));
}
firstUpgradeVersion = currentVersion + 1;
}
return IntStream.rangeClosed(firstUpgradeVersion, latestVersion)
.boxed()
.collect(toImmutableList());
}
}

View File

@@ -0,0 +1,43 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.schema;
import com.google.gerrit.server.config.AllProjectsName;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Inject;
import com.google.inject.Singleton;
/**
* Schema upgrade implementation.
*
* <p>Implementations must define a single public constructor that takes an {@link Arguments}. The
* recommended idiom is to pull out whichever individual fields from the {@code Arguments} are
* required by this implementation.
*/
interface NoteDbSchemaVersion {
@Singleton
class Arguments {
final GitRepositoryManager repoManager;
final AllProjectsName allProjects;
@Inject
Arguments(GitRepositoryManager repoManager, AllProjectsName allProjects) {
this.repoManager = repoManager;
this.allProjects = allProjects;
}
}
void upgrade(UpdateUI ui) throws Exception;
}

View File

@@ -0,0 +1,89 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.schema;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.notedb.NoteDbSchemaVersionManager;
import com.google.gerrit.server.notedb.NotesMigration;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
import com.google.inject.Module;
import com.google.inject.ProvisionException;
public class NoteDbSchemaVersionCheck implements LifecycleListener {
public static Module module() {
return new LifecycleModule() {
@Override
protected void configure() {
listener().to(NoteDbSchemaVersionCheck.class);
}
};
}
private final NotesMigration notesMigration;
private final NoteDbSchemaVersionManager versionManager;
private final SitePaths sitePaths;
@Inject
NoteDbSchemaVersionCheck(
NotesMigration notesMigration,
NoteDbSchemaVersionManager versionManager,
SitePaths sitePaths) {
this.notesMigration = notesMigration;
this.versionManager = versionManager;
this.sitePaths = sitePaths;
}
@Override
public void start() {
if (!notesMigration.commitChangeWrites()) {
// TODO(dborowitz): Only necessary to make migration tests pass; remove when NoteDb is the
// only option.
return;
}
try {
int current = versionManager.read();
if (current == 0) {
throw new ProvisionException(
String.format(
"Schema not yet initialized. Run init to initialize the schema:\n"
+ "$ java -jar gerrit.war init -d %s",
sitePaths.site_path.toAbsolutePath()));
}
int expected = NoteDbSchemaVersions.LATEST;
if (current != expected) {
String advice =
current > expected
? "Downgrade is not supported"
: String.format(
"Run init to upgrade:\n$ java -jar %s init -d %s",
sitePaths.gerrit_war.toAbsolutePath(), sitePaths.site_path.toAbsolutePath());
throw new ProvisionException(
String.format(
"Unsupported schema version %d; expected schema version %d. %s",
current, expected, advice));
}
} catch (OrmException e) {
throw new ProvisionException("Failed to read NoteDb schema version", e);
}
}
@Override
public void stop() {
// Do nothing.
}
}

View File

@@ -0,0 +1,62 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.schema;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap;
import static java.util.Comparator.naturalOrder;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.primitives.Ints;
import com.google.gerrit.server.UsedAt;
import java.lang.reflect.InvocationTargetException;
import java.util.Optional;
import java.util.stream.Stream;
public class NoteDbSchemaVersions {
static final ImmutableSortedMap<Integer, Class<? extends NoteDbSchemaVersion>> ALL =
// List all supported NoteDb schema versions here.
Stream.of(Schema_180.class)
.collect(toImmutableSortedMap(naturalOrder(), v -> guessVersion(v).get(), v -> v));
public static final int FIRST = ALL.firstKey();
public static final int LATEST = ALL.lastKey();
// TODO(dborowitz): Migrate delete-project plugin to use this implementation.
@UsedAt(UsedAt.Project.PLUGIN_DELETE_PROJECT)
public static Optional<Integer> guessVersion(Class<?> c) {
String prefix = "Schema_";
if (!c.getSimpleName().startsWith(prefix)) {
return Optional.empty();
}
return Optional.ofNullable(Ints.tryParse(c.getSimpleName().substring(prefix.length())));
}
public static NoteDbSchemaVersion get(
ImmutableSortedMap<Integer, Class<? extends NoteDbSchemaVersion>> schemaVersions,
int i,
NoteDbSchemaVersion.Arguments args) {
Class<? extends NoteDbSchemaVersion> clazz = schemaVersions.get(i);
checkArgument(clazz != null, "Schema version not found: %s", i);
try {
return clazz.getDeclaredConstructor(NoteDbSchemaVersion.Arguments.class).newInstance(args);
} catch (InstantiationException
| IllegalAccessException
| NoSuchMethodException
| InvocationTargetException e) {
throw new IllegalStateException("failed to invoke constructor on " + clazz.getName(), e);
}
}
}

View File

@@ -14,6 +14,8 @@
package com.google.gerrit.server.schema;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
@@ -36,8 +38,17 @@ import java.util.concurrent.TimeUnit;
/** A version of the database schema. */
public abstract class ReviewDbSchemaVersion {
/** The current schema version. */
// DO NOT upgrade this version in the master branch. Future versions must all be implemented as
// NoteDbSchemaVersions. It may be upgraded on the stable-2.16 branch, in which case this will
// need to be updated upon merging. In any case, this number must not exceed the first NoteDb
// schema version (180).
public static final Class<Schema_170> C = Schema_170.class;
static {
checkState(C.equals(Schema_170.class));
checkState(guessVersion(C) < 180);
}
public static int getBinaryVersion() {
return guessVersion(C);
}

View File

@@ -0,0 +1,27 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.schema;
public class Schema_180 implements NoteDbSchemaVersion {
@SuppressWarnings("unused")
Schema_180(Arguments args) {
// Do nothing.
}
@Override
public void upgrade(UpdateUI ui) {
// Do nothing; only used to populate the version ref, which is done by the caller.
}
}

View File

@@ -0,0 +1,79 @@
package com.google.gerrit.server.notedb;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assert_;
import static com.google.gerrit.reviewdb.client.RefNames.REFS_VERSION;
import com.google.gerrit.server.config.AllProjectsName;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.testing.GerritBaseTests;
import com.google.gerrit.testing.InMemoryRepositoryManager;
import com.google.gwtorm.server.OrmException;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.ObjectId;
import org.junit.Before;
import org.junit.Test;
public class NoteDbSchemaVersionManagerTest extends GerritBaseTests {
private NoteDbSchemaVersionManager manager;
private TestRepository<?> tr;
@Before
public void setUp() throws Exception {
AllProjectsName allProjectsName = new AllProjectsName("The-Projects");
GitRepositoryManager repoManager = new InMemoryRepositoryManager();
tr = new TestRepository<>(repoManager.createRepository(allProjectsName));
manager = new NoteDbSchemaVersionManager(allProjectsName, repoManager);
}
@Test
public void readMissing() throws Exception {
assertThat(manager.read()).isEqualTo(0);
}
@Test
public void read() throws Exception {
tr.update(REFS_VERSION, tr.blob("123"));
assertThat(manager.read()).isEqualTo(123);
}
@Test
public void readInvalid() throws Exception {
ObjectId blobId = tr.blob(" 1 2 3 ");
tr.update(REFS_VERSION, blobId);
try {
manager.read();
assert_().fail("expected OrmException");
} catch (OrmException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("invalid value in refs/meta/version blob at " + blobId.name());
}
}
@Test
public void incrementFromMissing() throws Exception {
manager.increment(123);
assertThat(manager.read()).isEqualTo(124);
}
@Test
public void increment() throws Exception {
tr.update(REFS_VERSION, tr.blob("123"));
manager.increment(123);
assertThat(manager.read()).isEqualTo(124);
}
@Test
public void incrementWrongOldVersion() throws Exception {
tr.update(REFS_VERSION, tr.blob("123"));
try {
manager.increment(456);
assert_().fail("expected OrmException");
} catch (OrmException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Expected old version 456 for refs/meta/version, found 123");
}
}
}

View File

@@ -0,0 +1,198 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.schema;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assert_;
import static com.google.common.truth.Truth8.assertThat;
import static com.google.gerrit.server.schema.NoteDbSchemaUpdater.requiredUpgrades;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.ImmutableSortedSet;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.server.config.AllProjectsName;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.notedb.IntBlob;
import com.google.gerrit.server.notedb.MutableNotesMigration;
import com.google.gerrit.server.notedb.NoteDbSchemaVersionManager;
import com.google.gerrit.server.notedb.NotesMigrationState;
import com.google.gerrit.testing.GerritBaseTests;
import com.google.gerrit.testing.InMemoryRepositoryManager;
import com.google.gerrit.testing.TestUpdateUI;
import com.google.gwtorm.server.OrmException;
import java.util.Optional;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Repository;
import org.junit.Test;
public class NoteDbSchemaUpdaterTest extends GerritBaseTests {
@Test
public void requiredUpgradesFromNoVersion() throws Exception {
assertThat(requiredUpgrades(0, versions(10))).containsExactly(10).inOrder();
assertThat(requiredUpgrades(0, versions(10, 11, 12))).containsExactly(10, 11, 12).inOrder();
}
@Test
public void requiredUpgradesFromExistingVersion() throws Exception {
ImmutableSortedSet<Integer> versions = versions(10, 11, 12, 13);
assertThat(requiredUpgrades(10, versions)).containsExactly(11, 12, 13).inOrder();
assertThat(requiredUpgrades(11, versions)).containsExactly(12, 13).inOrder();
assertThat(requiredUpgrades(12, versions)).containsExactly(13).inOrder();
assertThat(requiredUpgrades(13, versions)).isEmpty();
}
@Test
public void downgradeNotSupported() throws Exception {
try {
requiredUpgrades(14, versions(10, 11, 12, 13));
assert_().fail("expected OrmException");
} catch (OrmException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Cannot downgrade NoteDb schema from version 14 to 13");
}
}
@Test
public void skipToFirstVersionNotSupported() throws Exception {
ImmutableSortedSet<Integer> versions = versions(10, 11, 12);
assertThat(requiredUpgrades(9, versions)).containsExactly(10, 11, 12).inOrder();
try {
requiredUpgrades(8, versions);
assert_().fail("expected OrmException");
} catch (OrmException e) {
assertThat(e).hasMessageThat().isEqualTo("Cannot skip NoteDb schema from version 8 to 10");
}
}
private static class TestUpdate {
private final AllProjectsName allProjectsName;
private final NoteDbSchemaUpdater updater;
private final GitRepositoryManager repoManager;
private final NoteDbSchemaVersion.Arguments args;
TestUpdate(Optional<Integer> initialVersion) throws Exception {
allProjectsName = new AllProjectsName("The-Projects");
repoManager = new InMemoryRepositoryManager();
try (Repository repo = repoManager.createRepository(allProjectsName)) {
if (initialVersion.isPresent()) {
TestRepository<?> tr = new TestRepository<>(repo);
tr.update(RefNames.REFS_VERSION, tr.blob(initialVersion.get().toString()));
}
}
args = new NoteDbSchemaVersion.Arguments(repoManager, allProjectsName);
NoteDbSchemaVersionManager versionManager =
new NoteDbSchemaVersionManager(allProjectsName, repoManager);
MutableNotesMigration notesMigration = MutableNotesMigration.newDisabled();
notesMigration.setFrom(NotesMigrationState.NOTE_DB);
updater =
new NoteDbSchemaUpdater(
notesMigration,
versionManager,
args,
ImmutableSortedMap.of(10, TestSchema_10.class, 11, TestSchema_11.class));
}
ImmutableList<String> update() throws Exception {
ImmutableList.Builder<String> messages = ImmutableList.builder();
updater.update(
new TestUpdateUI() {
@Override
public void message(String m) {
messages.add(m);
}
});
return messages.build();
}
Optional<Integer> readVersion() throws Exception {
try (Repository repo = repoManager.openRepository(allProjectsName)) {
return IntBlob.parse(repo, RefNames.REFS_VERSION).map(IntBlob::value);
}
}
private static class TestSchema_10 implements NoteDbSchemaVersion {
@SuppressWarnings("unused")
TestSchema_10(Arguments args) {
// Do nothing.
}
@Override
public void upgrade(UpdateUI ui) {
ui.message("body of 10");
}
}
private static class TestSchema_11 implements NoteDbSchemaVersion {
@SuppressWarnings("unused")
TestSchema_11(Arguments args) {
// Do nothing.
}
@Override
public void upgrade(UpdateUI ui) {
ui.message("BODY OF 11");
}
}
}
@Test
public void bootstrapUpdate() throws Exception {
TestUpdate u = new TestUpdate(Optional.empty());
assertThat(u.update())
.containsExactly(
"Migrating data to schema 10 ...",
"body of 10",
"Migrating data to schema 11 ...",
"BODY OF 11")
.inOrder();
assertThat(u.readVersion()).hasValue(11);
}
@Test
public void updateTwoVersions() throws Exception {
TestUpdate u = new TestUpdate(Optional.of(9));
assertThat(u.update())
.containsExactly(
"Migrating data to schema 10 ...",
"body of 10",
"Migrating data to schema 11 ...",
"BODY OF 11")
.inOrder();
assertThat(u.readVersion()).hasValue(11);
}
@Test
public void updateOneVersion() throws Exception {
TestUpdate u = new TestUpdate(Optional.of(10));
assertThat(u.update())
.containsExactly("Migrating data to schema 11 ...", "BODY OF 11")
.inOrder();
assertThat(u.readVersion()).hasValue(11);
}
@Test
public void updateNoOp() throws Exception {
TestUpdate u = new TestUpdate(Optional.of(11));
assertThat(u.update()).isEmpty();
assertThat(u.readVersion()).hasValue(11);
}
private static ImmutableSortedSet<Integer> versions(Integer... versions) {
return ImmutableSortedSet.copyOf(versions);
}
}

View File

@@ -0,0 +1,78 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.schema;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static com.google.gerrit.server.schema.NoteDbSchemaVersions.guessVersion;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Streams;
import com.google.common.reflect.ClassPath;
import com.google.common.reflect.ClassPath.ClassInfo;
import com.google.gerrit.testing.GerritBaseTests;
import java.util.stream.IntStream;
import org.junit.Test;
public class NoteDbSchemaVersionsTest extends GerritBaseTests {
@Test
public void testGuessVersion() {
assertThat(guessVersion(getClass())).isEmpty();
assertThat(guessVersion(Schema_180.class)).hasValue(180);
}
@Test
public void contiguousVersions() {
ImmutableSortedSet<Integer> keys = NoteDbSchemaVersions.ALL.keySet();
ImmutableList<Integer> expected =
IntStream.rangeClosed(keys.first(), keys.last()).boxed().collect(toImmutableList());
assertThat(keys).containsExactlyElementsIn(expected).inOrder();
}
@Test
public void exceedsReviewDbVersion() {
assertThat(NoteDbSchemaVersions.ALL.firstKey())
// TODO(dborowitz): Replace with hard-coded max number once ReviewDb code is deleted.
.isGreaterThan(ReviewDbSchemaVersion.guessVersion(ReviewDbSchemaVersion.C));
}
@Test
public void containsAllNoteDbSchemas() throws Exception {
int minNoteDbVersion = 180;
ImmutableList<Integer> allSchemaVersions =
ClassPath.from(getClass().getClassLoader())
.getTopLevelClasses(getClass().getPackage().getName())
.stream()
.map(ClassInfo::load)
.map(NoteDbSchemaVersions::guessVersion)
.flatMap(Streams::stream)
.filter(v -> v >= minNoteDbVersion)
.sorted()
.collect(toImmutableList());
assertThat(NoteDbSchemaVersions.ALL.keySet())
.containsExactlyElementsIn(allSchemaVersions)
.inOrder();
}
@Test
public void schemaConstructors() throws Exception {
NoteDbSchemaVersion.Arguments args = new NoteDbSchemaVersion.Arguments(null, null);
for (int version : NoteDbSchemaVersions.ALL.keySet()) {
NoteDbSchemaVersions.get(NoteDbSchemaVersions.ALL, version, args);
}
}
}