Add test for online index schema migration

Requires some refactoring of AbstractVersionManager and OnlineReindexer
to allow the test to provide a listener that is injected into Daemon.
Use a simple OnlineUpgradeListener interface, which may end up being
useful for things other than tests.

In addition, factor out a separate LifecycleListener for starting the
online upgrade process. This is not immediately necessary, but will be
used in the near future for the NoteDb migration to hook into. In fact,
this change started life as this minor refactoring, at which point I
realized we probably need tests to make sure I don't break it.

Change-Id: Ifcbcac689cf14137784a250f025df149c90f22ef
This commit is contained in:
Dave Borowitz
2017-06-27 09:18:21 -04:00
parent d782a4f896
commit a9c6832afd
14 changed files with 521 additions and 68 deletions

View File

@@ -14,15 +14,20 @@
package com.google.gerrit.lucene;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.IndexConfig;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.OnlineUpgrader;
import com.google.gerrit.server.index.SingleVersionModule;
import com.google.gerrit.server.index.VersionManager;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.group.GroupIndex;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.FactoryModuleBuilder;
@@ -30,30 +35,40 @@ import java.util.Map;
import org.apache.lucene.search.BooleanQuery;
import org.eclipse.jgit.lib.Config;
public class LuceneIndexModule extends LifecycleModule {
public class LuceneIndexModule extends AbstractModule {
public static LuceneIndexModule singleVersionAllLatest(int threads) {
return new LuceneIndexModule(ImmutableMap.<String, Integer>of(), threads);
return new LuceneIndexModule(ImmutableMap.<String, Integer>of(), threads, false);
}
public static LuceneIndexModule singleVersionWithExplicitVersions(
Map<String, Integer> versions, int threads) {
return new LuceneIndexModule(versions, threads);
return new LuceneIndexModule(versions, threads, false);
}
public static LuceneIndexModule latestVersionWithOnlineUpgrade() {
return new LuceneIndexModule(null, 0);
return new LuceneIndexModule(null, 0, true);
}
public static LuceneIndexModule latestVersionWithoutOnlineUpgrade() {
return new LuceneIndexModule(null, 0, false);
}
static boolean isInMemoryTest(Config cfg) {
return cfg.getBoolean("index", "lucene", "testInmemory", false);
}
private final int threads;
private final Map<String, Integer> singleVersions;
private final int threads;
private final boolean onlineUpgrade;
private LuceneIndexModule(Map<String, Integer> singleVersions, int threads) {
private LuceneIndexModule(
Map<String, Integer> singleVersions, int threads, boolean onlineUpgrade) {
if (singleVersions != null) {
checkArgument(!onlineUpgrade, "online upgrade is incompatible with single version map");
}
this.singleVersions = singleVersions;
this.threads = threads;
this.onlineUpgrade = onlineUpgrade;
}
@Override
@@ -87,10 +102,14 @@ public class LuceneIndexModule extends LifecycleModule {
return IndexConfig.fromConfig(cfg).separateChangeSubIndexes(true).build();
}
private static class MultiVersionModule extends LifecycleModule {
private class MultiVersionModule extends LifecycleModule {
@Override
public void configure() {
bind(VersionManager.class).to(LuceneVersionManager.class);
listener().to(LuceneVersionManager.class);
if (onlineUpgrade) {
listener().to(OnlineUpgrader.class);
}
}
}
}

View File

@@ -15,14 +15,15 @@
package com.google.gerrit.lucene;
import com.google.common.primitives.Ints;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.AbstractVersionManager;
import com.google.gerrit.server.index.GerritIndexStatus;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.IndexDefinition;
import com.google.gerrit.server.index.OnlineUpgradeListener;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.VersionManager;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
@@ -36,10 +37,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class LuceneVersionManager extends AbstractVersionManager implements LifecycleListener {
public class LuceneVersionManager extends VersionManager {
private static final Logger log = LoggerFactory.getLogger(LuceneVersionManager.class);
private static class Version<V> extends AbstractVersionManager.Version<V> {
private static class Version<V> extends VersionManager.Version<V> {
private final boolean exists;
private Version(Schema<V> schema, int version, boolean exists, boolean ready) {
@@ -56,22 +57,22 @@ public class LuceneVersionManager extends AbstractVersionManager implements Life
LuceneVersionManager(
@GerritServerConfig Config cfg,
SitePaths sitePaths,
DynamicSet<OnlineUpgradeListener> listeners,
Collection<IndexDefinition<?, ?, ?>> defs) {
super(cfg, sitePaths, defs);
super(sitePaths, listeners, defs, VersionManager.getOnlineUpgrade(cfg));
}
@Override
protected <V> boolean isDirty(
Collection<com.google.gerrit.server.index.AbstractVersionManager.Version<V>> inUse,
com.google.gerrit.server.index.AbstractVersionManager.Version<V> v) {
Collection<com.google.gerrit.server.index.VersionManager.Version<V>> inUse,
com.google.gerrit.server.index.VersionManager.Version<V> v) {
return !inUse.contains(v) && ((Version<V>) v).exists;
}
@Override
protected <K, V, I extends Index<K, V>>
TreeMap<Integer, AbstractVersionManager.Version<V>> scanVersions(
IndexDefinition<K, V, I> def, GerritIndexStatus cfg) {
TreeMap<Integer, AbstractVersionManager.Version<V>> versions = new TreeMap<>();
protected <K, V, I extends Index<K, V>> TreeMap<Integer, VersionManager.Version<V>> scanVersions(
IndexDefinition<K, V, I> def, GerritIndexStatus cfg) {
TreeMap<Integer, VersionManager.Version<V>> versions = new TreeMap<>();
for (Schema<V> schema : def.getSchemas().values()) {
// This part is Lucene-specific.
Path p = getDir(sitePaths, def.getName(), schema);