Rename RebuildNoteDb to MigrateToNoteDb and expand flags

Change-Id: I812bcedb07d8986d938b47d0a70bc1e89c7f05d6
This commit is contained in:
Dave Borowitz 2017-06-16 15:02:26 -04:00
parent e7042be5cb
commit fd6ddff55c
9 changed files with 59 additions and 36 deletions

View File

@ -1742,7 +1742,7 @@ Common unit suffixes of 'k', 'm', or 'g' are supported.
If `true` enable the automatic mixed mode
(see link:http://www.h2database.com/html/features.html#auto_mixed_mode[Automatic Mixed Mode]).
This enables concurrent access to the embedded H2 database from command line
utils (e.g. RebuildNoteDb).
utils (e.g. MigrateToNoteDb).
+
Default is `false`.

View File

@ -99,7 +99,7 @@ previous options, unless otherwise noted.
inaccurate results, and writing to NoteDb would compound the problem. +
Thus it is up to an admin of a previously-ReviewDb site to ensure
MigratePrimaryStorage has been run for all changes. Note that the current
implementation of the `rebuild-note-db` program does not do this. +
implementation of the `migrate-to-note-db` program does not do this. +
In this phase, it would be possible to delete the Changes tables out from
under a running server with no effect.
- `noteDb.changes.fuseUpdates=true`: Code and meta updates within a single
@ -112,25 +112,25 @@ previous options, unless otherwise noted.
== Migration
Once configuration options are set, migration to NoteDb is primarily
accomplished by running the `rebuild-note-db` program. Currently, this program
accomplished by running the `migrate-to-note-db` program. Currently, this program
bulk copies ReviewDb data into NoteDb, but leaves primary storage of these
changes in ReviewDb, so the site is runnable with
`noteDb.changes.{write,read}=true`, but ReviewDb is still required.
Eventually, `rebuild-note-db` will set primary storage to NoteDb for all
Eventually, `migrate-to-note-db` will set primary storage to NoteDb for all
changes by default, so a site will be able to stop using ReviewDb for changes
immediately after a successful run.
There is code in `PrimaryStorageMigrator.java` to migrate individual changes
from NoteDb primary to ReviewDb primary. This code is not intended to be used
except in the event of a critical bug in NoteDb primary changes in production.
It will likely never be used by `rebuild-note-db`, and in fact it's not
recommended to run `rebuild-note-db` until the code is stable enough that the
It will likely never be used by `migrate-to-note-db`, and in fact it's not
recommended to run `migrate-to-note-db` until the code is stable enough that the
reverse migration won't be necessary.
=== Zero-Downtime Multi-Master Migration
Single-master Gerrit sites can use `rebuild-note-db` on an offline site to
Single-master Gerrit sites can use `migrate-to-note-db` on an offline site to
rebuild NoteDb, but this doesn't work in a zero-downtime environment like
googlesource.com.

View File

@ -29,7 +29,7 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class RebuildNoteDbIT {
public class MigrateToNoteDbIT {
private String sitePath;
private StoredConfig gerritConfig;
@ -49,14 +49,14 @@ public class RebuildNoteDbIT {
@Test
public void rebuildEmptySiteStartingWithNoteDbDisabed() throws Exception {
assertNotesMigrationState(NotesMigrationState.REVIEW_DB);
runGerrit("RebuildNoteDb", "-d", sitePath, "--show-stack-trace");
runGerrit("MigrateToNoteDb", "-d", sitePath, "--show-stack-trace");
assertNotesMigrationState(NotesMigrationState.READ_WRITE_NO_SEQUENCE);
}
@Test
public void rebuildEmptySiteStartingWithNoteDbEnabled() throws Exception {
setNotesMigrationState(NotesMigrationState.READ_WRITE_NO_SEQUENCE);
runGerrit("RebuildNoteDb", "-d", sitePath, "--show-stack-trace");
runGerrit("MigrateToNoteDb", "-d", sitePath, "--show-stack-trace");
assertNotesMigrationState(NotesMigrationState.READ_WRITE_NO_SEQUENCE);
}

View File

@ -29,7 +29,7 @@ import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.change.ChangeResource;
import com.google.gerrit.server.index.DummyIndexModule;
import com.google.gerrit.server.index.change.ReindexAfterRefUpdate;
import com.google.gerrit.server.notedb.rebuild.SiteRebuilder;
import com.google.gerrit.server.notedb.rebuild.NoteDbMigrator;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Provider;
@ -37,23 +37,46 @@ import java.util.ArrayList;
import java.util.List;
import org.kohsuke.args4j.Option;
public class RebuildNoteDb extends SiteProgram {
public class MigrateToNoteDb extends SiteProgram {
@Option(name = "--threads", usage = "Number of threads to use for rebuilding NoteDb")
private int threads = Runtime.getRuntime().availableProcessors();
@Option(name = "--project", usage = "Projects to rebuild; recommended for debugging only")
@Option(
name = "--project",
usage =
"Only rebuild these projects, do no other migration; incompatible with --change;"
+ " recommended for debugging only"
)
private List<String> projects = new ArrayList<>();
@Option(
name = "--change",
usage = "Individual change numbers to rebuild; recommended for debugging only"
usage =
"Only rebuild these changes, do no other migration; incompatible with --project;"
+ " recommended for debugging only"
)
private List<Integer> changes = new ArrayList<>();
@Option(
name = "--force",
usage =
"Force rebuilding changes where ReviewDb is still the source of truth, even if they"
+ " were previously migrated"
)
private boolean force;
@Option(
name = "--trial",
usage =
"trial mode: migrate changes and turn on reading from NoteDb, but leave ReviewDb as"
+ " the source of truth"
)
private boolean trial = true; // TODO(dborowitz): Default to false in 3.0.
private Injector dbInjector;
private Injector sysInjector;
@Inject private Provider<SiteRebuilder.Builder> rebuilderBuilderProvider;
@Inject private Provider<NoteDbMigrator.Builder> migratorBuilderProvider;
@Override
public int run() throws Exception {
@ -71,20 +94,20 @@ public class RebuildNoteDb extends SiteProgram {
sysManager.add(sysInjector);
sysManager.start();
try (SiteRebuilder rebuilder =
rebuilderBuilderProvider
try (NoteDbMigrator migrator =
migratorBuilderProvider
.get()
.setThreads(threads)
.setProgressOut(System.err)
.setTrialMode(true)
.setForceRebuild(true)
.setProjects(projects.stream().map(Project.NameKey::new).collect(toList()))
.setChanges(changes.stream().map(Change.Id::new).collect(toList()))
.setTrialMode(trial)
.setForceRebuild(force)
.build()) {
if (!projects.isEmpty() || !changes.isEmpty()) {
rebuilder.rebuild();
migrator.rebuild();
} else {
rebuilder.autoRebuild();
migrator.migrate();
}
}
return 0;

View File

@ -46,11 +46,11 @@ public class GerritServerIdProvider implements Provider<String> {
return;
}
// We're not generally supposed to do work in provider constructors, but
// this is a bit of a special case because we really need to have the ID
// available by the time the dbInjector is created. This even applies during
// RebuildNoteDb, which otherwise would have been a reasonable place to do
// the ID generation. Fortunately, it's not much work, and it happens once.
// We're not generally supposed to do work in provider constructors, but this is a bit of a
// special case because we really need to have the ID available by the time the dbInjector
// is created. This even applies during MigrateToNoteDb, which otherwise would have been a
// reasonable place to do the ID generation. Fortunately, it's not much work, and it happens
// once.
id = generate();
Config newCfg = readGerritConfig(sitePaths);
newCfg.setString(SECTION, null, KEY, id);

View File

@ -30,7 +30,7 @@ import org.eclipse.jgit.lib.Config;
* <p>This class controls the state of the migration according to options in {@code gerrit.config}.
* In general, any changes to these options should only be made by adventurous administrators, who
* know what they're doing, on non-production data, for the purposes of testing the NoteDb
* implementation. Changing options quite likely requires re-running {@code RebuildNoteDb}. For
* implementation. Changing options quite likely requires re-running {@code MigrateToNoteDb}. For
* these reasons, the options remain undocumented.
*/
@Singleton

View File

@ -40,7 +40,7 @@ import java.util.Objects;
* <p>This class controls the state of the migration according to options in {@code gerrit.config}.
* In general, any changes to these options should only be made by adventurous administrators, who
* know what they're doing, on non-production data, for the purposes of testing the NoteDb
* implementation. Changing options quite likely requires re-running {@code RebuildNoteDb}. For
* implementation. Changing options quite likely requires re-running {@code MigrateToNoteDb}. For
* these reasons, the options remain undocumented.
*
* <p><strong>Note:</strong> Callers should not assume the values returned by {@code

View File

@ -16,7 +16,7 @@ package com.google.gerrit.server.notedb.rebuild;
import java.io.IOException;
/** Exception thrown by {@link SiteRebuilder} when migration fails. */
/** Exception thrown by {@link NoteDbMigrator} when migration fails. */
class MigrationException extends IOException {
private static final long serialVersionUID = 1L;

View File

@ -70,9 +70,9 @@ import org.eclipse.jgit.util.io.NullOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** Rebuilder for all changes in a site. */
public class SiteRebuilder implements AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(SiteRebuilder.class);
/** One stop shop for migrating a site's change storage from ReviewDb to NoteDb. */
public class NoteDbMigrator implements AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(NoteDbMigrator.class);
public static class Builder {
private final SitePaths sitePaths;
@ -196,8 +196,8 @@ public class SiteRebuilder implements AutoCloseable {
return this;
}
public SiteRebuilder build() {
return new SiteRebuilder(
public NoteDbMigrator build() {
return new NoteDbMigrator(
sitePaths,
schemaFactory,
updateManagerFactory,
@ -227,7 +227,7 @@ public class SiteRebuilder implements AutoCloseable {
private final boolean trial;
private final boolean forceRebuild;
private SiteRebuilder(
private NoteDbMigrator(
SitePaths sitePaths,
SchemaFactory<ReviewDb> schemaFactory,
NoteDbUpdateManager.Factory updateManagerFactory,
@ -263,7 +263,7 @@ public class SiteRebuilder implements AutoCloseable {
executor.shutdownNow();
}
public void autoRebuild() throws OrmException, IOException {
public void migrate() throws OrmException, IOException {
checkState(
changes.isEmpty() && projects.isEmpty(),
"cannot set changes or projects during auto-migration; call rebuild() instead");