opendev/gerrit
Code Issues Proposed changes

Encapsulate per-index utilities in a pair of generic classes

Browse Source 
This collects the schema definitions (superseding the static
ChangeSchemas utilities) into a class called SchemaDefinitions.
Additionally, the backend-specific implementation classes (index
collection, site indexer, etc.) are collected into a class called
IndexDefinition. (We use the longer "definition" rather than "type"
because the latter is already used to describe the backend
implementation.)

The total set of supported index definitions is bound in IndexModule,
which provides them as a Collection<IndexDefinition<?, ?, ?>>.
Alternatively, callers that can't get the injected implementations
(notably InitIndex) can access the fully-static list of schema
definitions.

Abstract out LuceneChangeIndex.Factory into a generic IndexFactory
interface. At this point the only things that a particular backend
implementation needs to implement are:

 - The IndexFactory that constructs a per-implementation index
   implementation given a schema definition. Eventually this will
   expand to one IndexFactory implementation per index definition.
 - A listener that populates the IndexCollection for each supported
   index definition.

This change uses the Collection<IndexDefinition> in most places where
it is possible, with a few exceptions that will require a bit more
work:

 - LuceneVersionManager, which is used directly by some SSH commands
   to manipulate the index.
 - Reindex, which needs a different scheme for passing flags.

Change-Id: Ia5724cfecaae6335e7c0df24cd41c87b2bb5e36a
This commit is contained in:
Dave Borowitz
2016-03-17 14:00:49 +01:00
parent b518e69abc
commit 886746ff6f
18 changed files with 345 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@@ -123,10 +123,6 @@ public class LuceneChangeIndex implements ChangeIndex {
return QueryBuilder.intTerm(LEGACY_ID.getName(), id.get());
}
static interface Factory {
LuceneChangeIndex create(Schema<ChangeData> schema);
}
private final SitePaths sitePaths;
private final FillArgs fillArgs;
private final ListeningExecutorService executor;

97
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
View File

@@ -14,44 +14,69 @@
package com.google.gerrit.lucene;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.IndexConfig;
import com.google.gerrit.server.index.IndexDefinition;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.index.change.ChangeSchemas;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.inject.Inject;
import com.google.inject.Provides;
import com.google.inject.ProvisionException;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.name.Named;
import com.google.inject.name.Names;
import org.apache.lucene.search.BooleanQuery;
import org.eclipse.jgit.lib.Config;
import java.util.Collection;
import java.util.Map;
public class LuceneIndexModule extends LifecycleModule {
private static final String SINGLE_VERSIONS =
"LuceneIndexModule/SingleVersions";
public static LuceneIndexModule singleVersionAllLatest(int threads) {
return new LuceneIndexModule(ImmutableMap.<String, Integer> of(), threads);
}
public static LuceneIndexModule singleVersionWithExplicitVersions(
Map<String, Integer> versions, int threads) {
return new LuceneIndexModule(versions, threads);
}
public static LuceneIndexModule latestVersionWithOnlineUpgrade() {
return new LuceneIndexModule(null, 0);
}
static boolean isInMemoryTest(Config cfg) {
return cfg.getBoolean("index", "lucene", "testInmemory", false);
}
private final Integer singleVersion;
private final int threads;
private final Map<String, Integer> singleVersions;
public LuceneIndexModule() {
this(null, 0);
}
public LuceneIndexModule(Integer singleVersion, int threads) {
this.singleVersion = singleVersion;
private LuceneIndexModule(Map<String, Integer> singleVersions, int threads) {
this.singleVersions = singleVersions;
this.threads = threads;
}
@Override
protected void configure() {
factory(LuceneChangeIndex.Factory.class);
install(
new FactoryModuleBuilder()
.implement(ChangeIndex.class, LuceneChangeIndex.class)
.build(ChangeIndex.Factory.class));
install(new IndexModule(threads));
if (singleVersion == null) {
if (singleVersions == null) {
install(new MultiVersionModule());
} else {
install(new SingleVersionModule());
@@ -77,34 +102,48 @@ public class LuceneIndexModule extends LifecycleModule {
@Override
public void configure() {
listener().to(SingleVersionListener.class);
}
@Provides
@Singleton
LuceneChangeIndex getIndex(LuceneChangeIndex.Factory factory) {
Schema<ChangeData> schema = singleVersion != null
? ChangeSchemas.get(singleVersion)
: ChangeSchemas.getLatest();
return factory.create(schema);
bind(new TypeLiteral<Map<String, Integer>>() {})
.annotatedWith(Names.named(SINGLE_VERSIONS))
.toInstance(singleVersions);
}
}
@Singleton
static class SingleVersionListener implements LifecycleListener {
private final ChangeIndexCollection indexes;
private final LuceneChangeIndex index;
private final Collection<IndexDefinition<?, ?, ?>> defs;
private final Map<String, Integer> singleVersions;
@Inject
SingleVersionListener(ChangeIndexCollection indexes,
LuceneChangeIndex index) {
this.indexes = indexes;
this.index = index;
SingleVersionListener(
Collection<IndexDefinition<?, ?, ?>> defs,
@Named(SINGLE_VERSIONS) Map<String, Integer> singleVersions) {
this.defs = defs;
this.singleVersions = singleVersions;
}
@Override
public void start() {
indexes.setSearchIndex(index);
indexes.addWriteIndex(index);
for (IndexDefinition<?, ?, ?> def : defs) {
start(def);
}
}
private <K, V, I extends Index<K, V>> void start(
IndexDefinition<K, V, I> def) {
Schema<V> schema;
Integer v = singleVersions.get(def.getName());
if (v == null) {
schema = def.getLatest();
} else {
schema = def.getSchemas().get(v);
if (schema == null) {
throw new ProvisionException(String.format(
"Unrecognized %s schema version: %s", def.getName(), v));
}
}
I index = def.getIndexFactory().create(schema);
def.getIndexCollection().setSearchIndex(index);
def.getIndexCollection().addWriteIndex(index);
}
@Override

16
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
View File

@@ -25,10 +25,9 @@ import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.OnlineReindexer;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.change.AllChangesIndexer;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.index.change.ChangeIndexCollection;
import com.google.gerrit.server.index.change.ChangeSchemas;
import com.google.gerrit.server.index.change.ChangeIndexDefintion;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.inject.Inject;
import com.google.inject.ProvisionException;
@@ -97,7 +96,7 @@ public class LuceneVersionManager implements LifecycleListener {
private final SitePaths sitePaths;
private final LuceneChangeIndex.Factory indexFactory;
private final ChangeIndexCollection indexes;
private final AllChangesIndexer allChangesIndexer;
private final ChangeIndexDefintion changeDef;
private final boolean onlineUpgrade;
private OnlineReindexer<Change.Id, ChangeData, ChangeIndex> reindexer;
@@ -107,11 +106,11 @@ public class LuceneVersionManager implements LifecycleListener {
SitePaths sitePaths,
LuceneChangeIndex.Factory indexFactory,
ChangeIndexCollection indexes,
AllChangesIndexer allChangesIndexer) {
ChangeIndexDefintion changeDef) {
this.sitePaths = sitePaths;
this.indexFactory = indexFactory;
this.indexes = indexes;
this.allChangesIndexer = allChangesIndexer;
this.changeDef = changeDef;
this.onlineUpgrade = cfg.getBoolean("index", null, "onlineUpgrade", true);
}
@@ -161,7 +160,8 @@ public class LuceneVersionManager implements LifecycleListener {
}
markNotReady(cfg, versions.values(), write);
LuceneChangeIndex searchIndex = indexFactory.create(search.schema);
LuceneChangeIndex searchIndex =
(LuceneChangeIndex) indexFactory.create(search.schema);
indexes.setSearchIndex(searchIndex);
for (Version v : write) {
if (v.schema != null) {
@@ -175,7 +175,7 @@ public class LuceneVersionManager implements LifecycleListener {
int latest = write.get(0).version;
if (onlineUpgrade && latest != search.version) {
reindexer = new OnlineReindexer<>(indexes, allChangesIndexer, latest);
reindexer = new OnlineReindexer<>(changeDef, latest);
reindexer.start();
}
}
@@ -227,7 +227,7 @@ public class LuceneVersionManager implements LifecycleListener {
private TreeMap<Integer, Version> scanVersions(Config cfg) {
TreeMap<Integer, Version> versions = Maps.newTreeMap();
for (Schema<ChangeData> schema : ChangeSchemas.ALL.values()) {
for (Schema<ChangeData> schema : changeDef.getSchemas().values()) {
Path p = getDir(sitePaths, CHANGES_PREFIX, schema);
boolean isDir = Files.isDirectory(p);
if (Files.exists(p) && !isDir) {