opendev/gerrit
Code Issues Proposed changes

Introduce gerrit-index module

Browse Source 
gerrit-index module is meant to keep common parts for Lucene and
Elasticsearch indexes implementation.

Change-Id: Ie265dd9a262bdfec4b15e8abda885ffae7bb1aa4
Signed-off-by: Dariusz Luksza <dluksza@collab.net>
This commit is contained in:
Dariusz Luksza
2016-09-21 13:29:40 +02:00
committed by David Pursehouse
parent 8e72f5301b
commit 531906e16f
15 changed files with 211 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
gerrit-lucene/src/main/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
View File

@@ -25,6 +25,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gerrit.index.IndexUtils;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
@@ -51,7 +52,6 @@ import org.apache.lucene.search.ReferenceManager.RefreshListener;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,17 +75,6 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
return f.getName() + "_SORT";
}
public static void setReady(SitePaths sitePaths, String name, int version,
boolean ready) throws IOException {
try {
GerritIndexStatus cfg = new GerritIndexStatus(sitePaths);
cfg.setReady(name, version, ready);
cfg.save();
} catch (ConfigInvalidException e) {
throw new IOException(e);
}
}
private final Schema<V> schema;
private final SitePaths sitePaths;
private final Directory dir;
@@ -198,7 +187,7 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
@Override
public void markReady(boolean ready) throws IOException {
setReady(sitePaths, name, schema.getVersion(), ready);
IndexUtils.setReady(sitePaths, name, schema.getVersion(), ready);
}
@Override

78
gerrit-lucene/src/main/java/com/google/gerrit/lucene/GerritIndexStatus.java
View File

@@ -1,78 +0,0 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.lucene;
import com.google.common.primitives.Ints;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.util.FS;
import java.io.IOException;
class GerritIndexStatus {
private static final String SECTION = "index";
private static final String KEY_READY = "ready";
private final FileBasedConfig cfg;
GerritIndexStatus(SitePaths sitePaths)
throws ConfigInvalidException, IOException {
cfg = new FileBasedConfig(
sitePaths.index_dir.resolve("gerrit_index.config").toFile(),
FS.detect());
cfg.load();
convertLegacyConfig();
}
void setReady(String indexName, int version, boolean ready) {
cfg.setBoolean(SECTION, indexDirName(indexName, version), KEY_READY, ready);
}
boolean getReady(String indexName, int version) {
return cfg.getBoolean(SECTION, indexDirName(indexName, version), KEY_READY,
false);
}
void save() throws IOException {
cfg.save();
}
private void convertLegacyConfig() throws IOException {
boolean dirty = false;
// Convert legacy [index "25"] to modern [index "changes_0025"].
for (String subsection : cfg.getSubsections(SECTION)) {
Integer v = Ints.tryParse(subsection);
if (v != null) {
String ready = cfg.getString(SECTION, subsection, KEY_READY);
if (ready != null) {
dirty = false;
cfg.unset(SECTION, subsection, KEY_READY);
cfg.setString(SECTION,
indexDirName(ChangeSchemaDefinitions.NAME, v), KEY_READY, ready);
}
}
}
if (dirty) {
cfg.save();
}
}
private static String indexDirName(String indexName, int version) {
return String.format("%s_%04d", indexName, version);
}
}

25
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.gerrit.lucene.AbstractLuceneIndex.sortFieldName;
import static com.google.gerrit.lucene.LuceneVersionManager.CHANGES_PREFIX;
import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE;
import static com.google.gerrit.server.index.change.ChangeField.CHANGE;
import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
import static com.google.gerrit.server.index.change.ChangeIndexRewriter.CLOSED_STATUSES;
@@ -27,8 +26,6 @@ import static com.google.gerrit.server.index.change.ChangeIndexRewriter.OPEN_STA
import com.google.common.base.Throwables;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
@@ -36,6 +33,7 @@ import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.gerrit.index.IndexUtils;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.PatchSet;
@@ -92,7 +90,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -112,8 +109,6 @@ public class LuceneChangeIndex implements ChangeIndex {
public static final String CHANGES_OPEN = "open";
public static final String CHANGES_CLOSED = "closed";
public static final Map<String, String> CUSTOM_CHAR_MAPPING = ImmutableMap.of(
"_", " ", ".", " ");
static final String UPDATED_SORT_FIELD =
sortFieldName(ChangeField.UPDATED);
@@ -321,7 +316,7 @@ public class LuceneChangeIndex implements ChangeIndex {
throw new OrmException("interrupted");
}
final Set<String> fields = fields(opts);
final Set<String> fields = IndexUtils.fields(opts);
return new ChangeDataResults(
executor.submit(new Callable<List<Document>>() {
@Override
@@ -409,22 +404,6 @@ public class LuceneChangeIndex implements ChangeIndex {
}
}
public static Set<String> fields(QueryOptions opts) {
// Ensure we request enough fields to construct a ChangeData. We need both
// change ID and project, which can either come via the Change field or
// separate fields.
Set<String> fs = opts.fields();
if (fs.contains(CHANGE.getName())) {
// A Change is always sufficient.
return fs;
}
if (fs.contains(PROJECT.getName()) && fs.contains(LEGACY_ID.getName())) {
return fs;
}
return Sets.union(fs,
ImmutableSet.of(LEGACY_ID.getName(), PROJECT.getName()));
}
private static Multimap<String, IndexableField> fields(Document doc,
Set<String> fields) {
Multimap<String, IndexableField> stored =

84
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
View File

@@ -15,37 +15,23 @@
package com.google.gerrit.lucene;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.index.SingleVersionModule;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.IndexConfig;
import com.google.gerrit.server.index.IndexDefinition;
import com.google.gerrit.server.index.IndexModule;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.account.AccountIndex;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.inject.Inject;
import com.google.inject.Provides;
import com.google.inject.ProvisionException;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.name.Named;
import com.google.inject.name.Names;
import org.apache.lucene.search.BooleanQuery;
import org.eclipse.jgit.lib.Config;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
public class LuceneIndexModule extends LifecycleModule {
private static final String SINGLE_VERSIONS =
"LuceneIndexModule/SingleVersions";
public static LuceneIndexModule singleVersionAllLatest(int threads) {
return new LuceneIndexModule(ImmutableMap.<String, Integer> of(), threads);
}
@@ -104,72 +90,4 @@ public class LuceneIndexModule extends LifecycleModule {
listener().to(LuceneVersionManager.class);
}
}
public static class SingleVersionModule extends LifecycleModule {
private final Map<String, Integer> singleVersions;
public SingleVersionModule(Map<String, Integer> singleVersions) {
this.singleVersions = singleVersions;
}
@Override
public void configure() {
listener().to(SingleVersionListener.class);
bind(new TypeLiteral<Map<String, Integer>>() {})
.annotatedWith(Names.named(SINGLE_VERSIONS))
.toInstance(singleVersions);
}
}
@Singleton
private static class SingleVersionListener implements LifecycleListener {
private final Set<String> disabled;
private final Collection<IndexDefinition<?, ?, ?>> defs;
private final Map<String, Integer> singleVersions;
@Inject
SingleVersionListener(
@GerritServerConfig Config cfg,
Collection<IndexDefinition<?, ?, ?>> defs,
@Named(SINGLE_VERSIONS) Map<String, Integer> singleVersions) {
this.defs = defs;
this.singleVersions = singleVersions;
disabled = ImmutableSet.copyOf(
cfg.getStringList("index", null, "testDisable"));
}
@Override
public void start() {
for (IndexDefinition<?, ?, ?> def : defs) {
start(def);
}
}
private <K, V, I extends Index<K, V>> void start(
IndexDefinition<K, V, I> def) {
if (disabled.contains(def.getName())) {
return;
}
Schema<V> schema;
Integer v = singleVersions.get(def.getName());
if (v == null) {
schema = def.getLatest();
} else {
schema = def.getSchemas().get(v);
if (schema == null) {
throw new ProvisionException(String.format(
"Unrecognized %s schema version: %s", def.getName(), v));
}
}
I index = def.getIndexFactory().create(schema);
def.getIndexCollection().setSearchIndex(index);
def.getIndexCollection().addWriteIndex(index);
}
@Override
public void stop() {
// Do nothing; indexes are closed by IndexCollection.
}
}
}

1
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java
View File

@@ -20,6 +20,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.primitives.Ints;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.index.GerritIndexStatus;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.Index;