diff --git a/java/com/google/gerrit/pgm/MigrateToNoteDb.java b/java/com/google/gerrit/pgm/MigrateToNoteDb.java index 61d7ed91cf..09422b2193 100644 --- a/java/com/google/gerrit/pgm/MigrateToNoteDb.java +++ b/java/com/google/gerrit/pgm/MigrateToNoteDb.java @@ -60,15 +60,20 @@ public class MigrateToNoteDb extends SiteProgram { @Option( name = "--project", usage = - "Only rebuild these projects, do no other migration; incompatible with --change;" - + " recommended for debugging only") + "Only rebuild these projects, do no other migration; incompatible with --change" + + " and --skip-project; recommended for debugging only") private List projects = new ArrayList<>(); + @Option( + name = "--skip-project", + usage = "Rebuild all projects except these; incompatible with the --project and --change") + private List skipProjects = new ArrayList<>(); + @Option( name = "--change", usage = - "Only rebuild these changes, do no other migration; incompatible with --project;" - + " recommended for debugging only") + "Only rebuild these changes, do no other migration; incompatible with --project and" + + " --skip-project; recommended for debugging only") private List changes = new ArrayList<>(); @Option( @@ -132,12 +137,13 @@ public class MigrateToNoteDb extends SiteProgram { .setThreads(threads) .setProgressOut(System.err) .setProjects(projects.stream().map(Project.NameKey::new).collect(toList())) + .setSkipProjects(skipProjects.stream().map(Project.NameKey::new).collect(toList())) .setChanges(changes.stream().map(Change.Id::new).collect(toList())) .setTrialMode(trial) .setForceRebuild(force) .setSequenceGap(sequenceGap) .build()) { - if (!projects.isEmpty() || !changes.isEmpty()) { + if (!projects.isEmpty() || !changes.isEmpty() || !skipProjects.isEmpty()) { migrator.rebuild(); } else { migrator.migrate(); diff --git a/java/com/google/gerrit/server/api/changes/ChangeEditApiImpl.java b/java/com/google/gerrit/server/api/changes/ChangeEditApiImpl.java index 73f6740e02..8b9ffaf262 100644 --- a/java/com/google/gerrit/server/api/changes/ChangeEditApiImpl.java +++ b/java/com/google/gerrit/server/api/changes/ChangeEditApiImpl.java @@ -35,6 +35,7 @@ import com.google.gerrit.server.restapi.change.PublishChangeEdit; import com.google.gerrit.server.restapi.change.RebaseChangeEdit; import com.google.gwtorm.server.OrmException; import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.assistedinject.Assisted; import java.io.IOException; import java.util.Optional; @@ -44,42 +45,42 @@ public class ChangeEditApiImpl implements ChangeEditApi { ChangeEditApiImpl create(ChangeResource changeResource); } - private final ChangeEdits.Detail editDetail; + private final Provider editDetailProvider; private final ChangeEdits.Post changeEditsPost; private final DeleteChangeEdit deleteChangeEdit; private final RebaseChangeEdit rebaseChangeEdit; private final PublishChangeEdit publishChangeEdit; - private final ChangeEdits.Get changeEditsGet; + private final Provider changeEditsGetProvider; private final ChangeEdits.Put changeEditsPut; private final ChangeEdits.DeleteContent changeEditDeleteContent; - private final ChangeEdits.GetMessage getChangeEditCommitMessage; + private final Provider getChangeEditCommitMessageProvider; private final ChangeEdits.EditMessage modifyChangeEditCommitMessage; private final ChangeEdits changeEdits; private final ChangeResource changeResource; @Inject public ChangeEditApiImpl( - ChangeEdits.Detail editDetail, + Provider editDetailProvider, ChangeEdits.Post changeEditsPost, DeleteChangeEdit deleteChangeEdit, RebaseChangeEdit rebaseChangeEdit, PublishChangeEdit publishChangeEdit, - ChangeEdits.Get changeEditsGet, + Provider changeEditsGetProvider, ChangeEdits.Put changeEditsPut, ChangeEdits.DeleteContent changeEditDeleteContent, - ChangeEdits.GetMessage getChangeEditCommitMessage, + Provider getChangeEditCommitMessageProvider, ChangeEdits.EditMessage modifyChangeEditCommitMessage, ChangeEdits changeEdits, @Assisted ChangeResource changeResource) { - this.editDetail = editDetail; + this.editDetailProvider = editDetailProvider; this.changeEditsPost = changeEditsPost; this.deleteChangeEdit = deleteChangeEdit; this.rebaseChangeEdit = rebaseChangeEdit; this.publishChangeEdit = publishChangeEdit; - this.changeEditsGet = changeEditsGet; + this.changeEditsGetProvider = changeEditsGetProvider; this.changeEditsPut = changeEditsPut; this.changeEditDeleteContent = changeEditDeleteContent; - this.getChangeEditCommitMessage = getChangeEditCommitMessage; + this.getChangeEditCommitMessageProvider = getChangeEditCommitMessageProvider; this.modifyChangeEditCommitMessage = modifyChangeEditCommitMessage; this.changeEdits = changeEdits; this.changeResource = changeResource; @@ -88,7 +89,7 @@ public class ChangeEditApiImpl implements ChangeEditApi { @Override public Optional get() throws RestApiException { try { - Response edit = editDetail.apply(changeResource); + Response edit = editDetailProvider.get().apply(changeResource); return edit.isNone() ? Optional.empty() : Optional.of(edit.value()); } catch (Exception e) { throw asRestApiException("Cannot retrieve change edit", e); @@ -140,7 +141,7 @@ public class ChangeEditApiImpl implements ChangeEditApi { public Optional getFile(String filePath) throws RestApiException { try { ChangeEditResource changeEditResource = getChangeEditResource(filePath); - Response fileResponse = changeEditsGet.apply(changeEditResource); + Response fileResponse = changeEditsGetProvider.get().apply(changeEditResource); return fileResponse.isNone() ? Optional.empty() : Optional.of(fileResponse.value()); } catch (Exception e) { throw asRestApiException("Cannot retrieve file of change edit", e); @@ -191,7 +192,8 @@ public class ChangeEditApiImpl implements ChangeEditApi { @Override public String getCommitMessage() throws RestApiException { try { - try (BinaryResult binaryResult = getChangeEditCommitMessage.apply(changeResource)) { + try (BinaryResult binaryResult = + getChangeEditCommitMessageProvider.get().apply(changeResource)) { return binaryResult.asString(); } } catch (Exception e) { diff --git a/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java b/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java index 225926d69b..736792df5c 100644 --- a/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java +++ b/java/com/google/gerrit/server/notedb/rebuild/NoteDbMigrator.java @@ -154,6 +154,7 @@ public class NoteDbMigrator implements AutoCloseable { private int threads; private ImmutableList projects = ImmutableList.of(); + private ImmutableList skipProjects = ImmutableList.of(); private ImmutableList changes = ImmutableList.of(); private OutputStream progressOut = NullOutputStream.INSTANCE; private NotesMigrationState stopAtState; @@ -235,6 +236,22 @@ public class NoteDbMigrator implements AutoCloseable { return this; } + /** + * Process all projects except these + * + *

Incompatible with {@link #setProjects(Collection)} and {@link #setChanges(Collection)} + * + *

By default, all projects will be processed. + * + * @param skipProjects set of projects; if null or empty all project will be processed + * @return this. + */ + public Builder setSkipProjects(@Nullable Collection skipProjects) { + this.skipProjects = + skipProjects != null ? ImmutableList.copyOf(skipProjects) : ImmutableList.of(); + return this; + } + /** * Limit the set of changes that are processed. * @@ -366,6 +383,7 @@ public class NoteDbMigrator implements AutoCloseable { workQueue.createQueue(threads, "RebuildChange", true)) : MoreExecutors.newDirectExecutorService(), projects, + skipProjects, changes, progressOut, stopAtState, @@ -394,6 +412,7 @@ public class NoteDbMigrator implements AutoCloseable { private final ListeningExecutorService executor; private final ImmutableList projects; + private final ImmutableList skipProjects; private final ImmutableList changes; private final OutputStream progressOut; private final NotesMigrationState stopAtState; @@ -419,6 +438,7 @@ public class NoteDbMigrator implements AutoCloseable { DynamicSet listeners, ListeningExecutorService executor, ImmutableList projects, + ImmutableList skipProjects, ImmutableList changes, OutputStream progressOut, NotesMigrationState stopAtState, @@ -427,8 +447,12 @@ public class NoteDbMigrator implements AutoCloseable { int sequenceGap, boolean autoMigrate) throws MigrationException { - if (!changes.isEmpty() && !projects.isEmpty()) { - throw new MigrationException("Cannot set both changes and projects"); + if (ImmutableList.of(!changes.isEmpty(), !projects.isEmpty(), !skipProjects.isEmpty()) + .stream() + .filter(e -> e) + .count() + > 1) { + throw new MigrationException("Cannot combine changes, projects and skipProjects"); } if (sequenceGap < 0) { throw new MigrationException("Sequence gap must be non-negative: " + sequenceGap); @@ -449,6 +473,7 @@ public class NoteDbMigrator implements AutoCloseable { this.listeners = listeners; this.executor = executor; this.projects = projects; + this.skipProjects = skipProjects; this.changes = changes; this.progressOut = progressOut; this.stopAtState = stopAtState; @@ -469,9 +494,9 @@ public class NoteDbMigrator implements AutoCloseable { } public void migrate() throws OrmException, IOException { - if (!changes.isEmpty() || !projects.isEmpty()) { + if (!changes.isEmpty() || !projects.isEmpty() || !skipProjects.isEmpty()) { throw new MigrationException( - "Cannot set changes or projects during full migration; call rebuild() instead"); + "Cannot set changes or projects or skipProjects during full migration; call rebuild() instead"); } Optional maybeState = loadState(); if (!maybeState.isPresent()) { @@ -582,7 +607,7 @@ public class NoteDbMigrator implements AutoCloseable { private NotesMigrationState setNoteDbPrimary(NotesMigrationState prev) throws MigrationException, OrmException, IOException { checkState( - projects.isEmpty() && changes.isEmpty(), + projects.isEmpty() && changes.isEmpty() && skipProjects.isEmpty(), "Should not have attempted setNoteDbPrimary with a subset of changes"); checkState( prev == READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY @@ -804,6 +829,9 @@ public class NoteDbMigrator implements AutoCloseable { if (!projects.isEmpty()) { return byProject(db.changes().all(), c -> projects.contains(c.getProject()), out); } + if (!skipProjects.isEmpty()) { + return byProject(db.changes().all(), c -> !skipProjects.contains(c.getProject()), out); + } if (!changes.isEmpty()) { return byProject(db.changes().get(changes), c -> true, out); } diff --git a/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java b/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java index 87c5ace12d..468e8103d5 100644 --- a/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java +++ b/javatests/com/google/gerrit/acceptance/server/notedb/OnlineNoteDbMigrationIT.java @@ -141,15 +141,29 @@ public class OnlineNoteDbMigrationIT extends AbstractDaemonTest { assertMigrationException( "Cannot rebuild without noteDb.changes.write=true", b -> b, NoteDbMigrator::rebuild); assertMigrationException( - "Cannot set both changes and projects", b -> b.setChanges(cs).setProjects(ps), m -> {}); + "Cannot combine changes, projects and skipProjects", + b -> b.setChanges(cs).setProjects(ps), + m -> {}); assertMigrationException( - "Cannot set changes or projects during full migration", + "Cannot combine changes, projects and skipProjects", + b -> b.setChanges(cs).setSkipProjects(ps), + m -> {}); + assertMigrationException( + "Cannot combine changes, projects and skipProjects", + b -> b.setProjects(ps).setSkipProjects(ps), + m -> {}); + assertMigrationException( + "Cannot set changes or projects or skipProjects during full migration", b -> b.setChanges(cs), NoteDbMigrator::migrate); assertMigrationException( - "Cannot set changes or projects during full migration", + "Cannot set changes or projects or skipProjects during full migration", b -> b.setProjects(ps), NoteDbMigrator::migrate); + assertMigrationException( + "Cannot set changes or projects or skipProjects during full migration", + b -> b.setSkipProjects(ps), + NoteDbMigrator::migrate); setNotesMigrationState(READ_WRITE_WITH_SEQUENCE_REVIEW_DB_PRIMARY); assertMigrationException( @@ -311,12 +325,11 @@ public class OnlineNoteDbMigrationIT extends AbstractDaemonTest { Change.Id id1 = r1.getChange().getId(); Change.Id id2 = r2.getChange().getId(); - String invalidState = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; try (ReviewDb db = schemaFactory.open()) { Change c1 = db.changes().get(id1); - c1.setNoteDbState(invalidState); + c1.setNoteDbState(INVALID_STATE); Change c2 = db.changes().get(id2); - c2.setNoteDbState(invalidState); + c2.setNoteDbState(INVALID_STATE); db.changes().update(ImmutableList.of(c1, c2)); } @@ -324,10 +337,50 @@ public class OnlineNoteDbMigrationIT extends AbstractDaemonTest { try (ReviewDb db = schemaFactory.open()) { NoteDbChangeState s1 = NoteDbChangeState.parse(db.changes().get(id1)); - assertThat(s1.getChangeMetaId().name()).isEqualTo(invalidState); + assertThat(s1.getChangeMetaId().name()).isEqualTo(INVALID_STATE); NoteDbChangeState s2 = NoteDbChangeState.parse(db.changes().get(id2)); - assertThat(s2.getChangeMetaId().name()).isNotEqualTo(invalidState); + assertThat(s2.getChangeMetaId().name()).isNotEqualTo(INVALID_STATE); + } + } + + @Test + public void rebuildNonSkippedProjects() throws Exception { + setNotesMigrationState(WRITE); + + Project.NameKey p2 = createProject("project2"); + TestRepository tr2 = cloneProject(p2, admin); + Project.NameKey p3 = createProject("project3"); + TestRepository tr3 = cloneProject(p3, admin); + + PushOneCommit.Result r1 = createChange(); + PushOneCommit.Result r2 = pushFactory.create(db, admin.getIdent(), tr2).to("refs/for/master"); + PushOneCommit.Result r3 = pushFactory.create(db, admin.getIdent(), tr3).to("refs/for/master"); + Change.Id id1 = r1.getChange().getId(); + Change.Id id2 = r2.getChange().getId(); + Change.Id id3 = r3.getChange().getId(); + + try (ReviewDb db = schemaFactory.open()) { + Change c1 = db.changes().get(id1); + c1.setNoteDbState(INVALID_STATE); + Change c2 = db.changes().get(id2); + c2.setNoteDbState(INVALID_STATE); + Change c3 = db.changes().get(id3); + c3.setNoteDbState(INVALID_STATE); + db.changes().update(ImmutableList.of(c1, c2, c3)); + } + + migrate(b -> b.setSkipProjects(ImmutableList.of(p3)), NoteDbMigrator::rebuild); + + try (ReviewDb db = schemaFactory.open()) { + NoteDbChangeState s1 = NoteDbChangeState.parse(db.changes().get(id1)); + assertThat(s1.getChangeMetaId().name()).isNotEqualTo(INVALID_STATE); + + NoteDbChangeState s2 = NoteDbChangeState.parse(db.changes().get(id2)); + assertThat(s2.getChangeMetaId().name()).isNotEqualTo(INVALID_STATE); + + NoteDbChangeState s3 = NoteDbChangeState.parse(db.changes().get(id3)); + assertThat(s3.getChangeMetaId().name()).isEqualTo(INVALID_STATE); } } diff --git a/plugins/download-commands b/plugins/download-commands index ce8f07234c..22495f7aaa 160000 --- a/plugins/download-commands +++ b/plugins/download-commands @@ -1 +1 @@ -Subproject commit ce8f07234c96e563cfc60aae33b64e2148e0e192 +Subproject commit 22495f7aaa9f91b55c0482cefe27bb117d1869c9