Move GerritIndexWriterConfig to its own file
Change-Id: I237b61bdda7017d524215773200e6f49c580f236
This commit is contained in:
@@ -0,0 +1,77 @@
|
|||||||
|
// Copyright (C) 2016 The Android Open Source Project
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package com.google.gerrit.lucene;
|
||||||
|
|
||||||
|
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||||
|
import static java.util.concurrent.TimeUnit.MINUTES;
|
||||||
|
|
||||||
|
import com.google.common.collect.ImmutableMap;
|
||||||
|
import com.google.gerrit.server.config.ConfigUtil;
|
||||||
|
|
||||||
|
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
||||||
|
import org.apache.lucene.analysis.util.CharArraySet;
|
||||||
|
import org.apache.lucene.index.IndexWriterConfig;
|
||||||
|
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
|
||||||
|
import org.eclipse.jgit.lib.Config;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Combination of Lucene {@link IndexWriterConfig} with additional
|
||||||
|
* Gerrit-specific options.
|
||||||
|
*/
|
||||||
|
class GerritIndexWriterConfig {
|
||||||
|
private static final Map<String, String> CUSTOM_CHAR_MAPPING =
|
||||||
|
ImmutableMap.of("_", " ", ".", " ");
|
||||||
|
|
||||||
|
private final IndexWriterConfig luceneConfig;
|
||||||
|
private long commitWithinMs;
|
||||||
|
private final CustomMappingAnalyzer analyzer;
|
||||||
|
|
||||||
|
GerritIndexWriterConfig(Config cfg, String name) {
|
||||||
|
analyzer =
|
||||||
|
new CustomMappingAnalyzer(new StandardAnalyzer(
|
||||||
|
CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING);
|
||||||
|
luceneConfig = new IndexWriterConfig(analyzer)
|
||||||
|
.setOpenMode(OpenMode.CREATE_OR_APPEND)
|
||||||
|
.setCommitOnClose(true);
|
||||||
|
double m = 1 << 20;
|
||||||
|
luceneConfig.setRAMBufferSizeMB(cfg.getLong(
|
||||||
|
"index", name, "ramBufferSize",
|
||||||
|
(long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB * m)) / m);
|
||||||
|
luceneConfig.setMaxBufferedDocs(cfg.getInt(
|
||||||
|
"index", name, "maxBufferedDocs",
|
||||||
|
IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS));
|
||||||
|
try {
|
||||||
|
commitWithinMs =
|
||||||
|
ConfigUtil.getTimeUnit(cfg, "index", name, "commitWithin",
|
||||||
|
MILLISECONDS.convert(5, MINUTES), MILLISECONDS);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
commitWithinMs = cfg.getLong("index", name, "commitWithin", 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
CustomMappingAnalyzer getAnalyzer() {
|
||||||
|
return analyzer;
|
||||||
|
}
|
||||||
|
|
||||||
|
IndexWriterConfig getLuceneConfig() {
|
||||||
|
return luceneConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
long getCommitWithinMs() {
|
||||||
|
return commitWithinMs;
|
||||||
|
}
|
||||||
|
}
|
@@ -22,10 +22,7 @@ import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
|
|||||||
import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
|
import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
|
||||||
import static com.google.gerrit.server.index.change.IndexRewriter.CLOSED_STATUSES;
|
import static com.google.gerrit.server.index.change.IndexRewriter.CLOSED_STATUSES;
|
||||||
import static com.google.gerrit.server.index.change.IndexRewriter.OPEN_STATUSES;
|
import static com.google.gerrit.server.index.change.IndexRewriter.OPEN_STATUSES;
|
||||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
|
||||||
import static java.util.concurrent.TimeUnit.MINUTES;
|
|
||||||
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.google.common.collect.Sets;
|
import com.google.common.collect.Sets;
|
||||||
@@ -37,7 +34,6 @@ import com.google.gerrit.reviewdb.client.Change;
|
|||||||
import com.google.gerrit.reviewdb.client.PatchSet;
|
import com.google.gerrit.reviewdb.client.PatchSet;
|
||||||
import com.google.gerrit.reviewdb.client.Project;
|
import com.google.gerrit.reviewdb.client.Project;
|
||||||
import com.google.gerrit.reviewdb.server.ReviewDb;
|
import com.google.gerrit.reviewdb.server.ReviewDb;
|
||||||
import com.google.gerrit.server.config.ConfigUtil;
|
|
||||||
import com.google.gerrit.server.config.GerritServerConfig;
|
import com.google.gerrit.server.config.GerritServerConfig;
|
||||||
import com.google.gerrit.server.config.SitePaths;
|
import com.google.gerrit.server.config.SitePaths;
|
||||||
import com.google.gerrit.server.index.FieldDef;
|
import com.google.gerrit.server.index.FieldDef;
|
||||||
@@ -64,8 +60,6 @@ import com.google.inject.Provider;
|
|||||||
import com.google.inject.assistedinject.Assisted;
|
import com.google.inject.assistedinject.Assisted;
|
||||||
import com.google.inject.assistedinject.AssistedInject;
|
import com.google.inject.assistedinject.AssistedInject;
|
||||||
|
|
||||||
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
|
||||||
import org.apache.lucene.analysis.util.CharArraySet;
|
|
||||||
import org.apache.lucene.document.Document;
|
import org.apache.lucene.document.Document;
|
||||||
import org.apache.lucene.document.Field;
|
import org.apache.lucene.document.Field;
|
||||||
import org.apache.lucene.document.Field.Store;
|
import org.apache.lucene.document.Field.Store;
|
||||||
@@ -76,8 +70,6 @@ import org.apache.lucene.document.StoredField;
|
|||||||
import org.apache.lucene.document.StringField;
|
import org.apache.lucene.document.StringField;
|
||||||
import org.apache.lucene.document.TextField;
|
import org.apache.lucene.document.TextField;
|
||||||
import org.apache.lucene.index.IndexWriter;
|
import org.apache.lucene.index.IndexWriter;
|
||||||
import org.apache.lucene.index.IndexWriterConfig;
|
|
||||||
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
|
|
||||||
import org.apache.lucene.index.IndexableField;
|
import org.apache.lucene.index.IndexableField;
|
||||||
import org.apache.lucene.index.Term;
|
import org.apache.lucene.index.Term;
|
||||||
import org.apache.lucene.search.BooleanQuery;
|
import org.apache.lucene.search.BooleanQuery;
|
||||||
@@ -105,7 +97,6 @@ import java.util.ArrayList;
|
|||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
|
|
||||||
@@ -137,9 +128,6 @@ public class LuceneChangeIndex implements ChangeIndex {
|
|||||||
private static final String ID_SORT_FIELD =
|
private static final String ID_SORT_FIELD =
|
||||||
sortFieldName(ChangeField.LEGACY_ID);
|
sortFieldName(ChangeField.LEGACY_ID);
|
||||||
|
|
||||||
private static final Map<String, String> CUSTOM_CHAR_MAPPING = ImmutableMap.of(
|
|
||||||
"_", " ", ".", " ");
|
|
||||||
|
|
||||||
public static void setReady(SitePaths sitePaths, int version, boolean ready)
|
public static void setReady(SitePaths sitePaths, int version, boolean ready)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
try {
|
try {
|
||||||
@@ -160,42 +148,6 @@ public class LuceneChangeIndex implements ChangeIndex {
|
|||||||
LuceneChangeIndex create(Schema<ChangeData> schema);
|
LuceneChangeIndex create(Schema<ChangeData> schema);
|
||||||
}
|
}
|
||||||
|
|
||||||
static class GerritIndexWriterConfig {
|
|
||||||
private final IndexWriterConfig luceneConfig;
|
|
||||||
private long commitWithinMs;
|
|
||||||
|
|
||||||
private GerritIndexWriterConfig(Config cfg, String name) {
|
|
||||||
CustomMappingAnalyzer analyzer =
|
|
||||||
new CustomMappingAnalyzer(new StandardAnalyzer(
|
|
||||||
CharArraySet.EMPTY_SET), CUSTOM_CHAR_MAPPING);
|
|
||||||
luceneConfig = new IndexWriterConfig(analyzer)
|
|
||||||
.setOpenMode(OpenMode.CREATE_OR_APPEND)
|
|
||||||
.setCommitOnClose(true);
|
|
||||||
double m = 1 << 20;
|
|
||||||
luceneConfig.setRAMBufferSizeMB(cfg.getLong(
|
|
||||||
"index", name, "ramBufferSize",
|
|
||||||
(long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB * m)) / m);
|
|
||||||
luceneConfig.setMaxBufferedDocs(cfg.getInt(
|
|
||||||
"index", name, "maxBufferedDocs",
|
|
||||||
IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS));
|
|
||||||
try {
|
|
||||||
commitWithinMs =
|
|
||||||
ConfigUtil.getTimeUnit(cfg, "index", name, "commitWithin",
|
|
||||||
MILLISECONDS.convert(5, MINUTES), MILLISECONDS);
|
|
||||||
} catch (IllegalArgumentException e) {
|
|
||||||
commitWithinMs = cfg.getLong("index", name, "commitWithin", 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
IndexWriterConfig getLuceneConfig() {
|
|
||||||
return luceneConfig;
|
|
||||||
}
|
|
||||||
|
|
||||||
long getCommitWithinMs() {
|
|
||||||
return commitWithinMs;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private final SitePaths sitePaths;
|
private final SitePaths sitePaths;
|
||||||
private final FillArgs fillArgs;
|
private final FillArgs fillArgs;
|
||||||
private final ListeningExecutorService executor;
|
private final ListeningExecutorService executor;
|
||||||
@@ -222,19 +174,16 @@ public class LuceneChangeIndex implements ChangeIndex {
|
|||||||
this.changeDataFactory = changeDataFactory;
|
this.changeDataFactory = changeDataFactory;
|
||||||
this.schema = schema;
|
this.schema = schema;
|
||||||
|
|
||||||
CustomMappingAnalyzer analyzer =
|
|
||||||
new CustomMappingAnalyzer(new StandardAnalyzer(CharArraySet.EMPTY_SET),
|
|
||||||
CUSTOM_CHAR_MAPPING);
|
|
||||||
queryBuilder = new QueryBuilder(analyzer);
|
|
||||||
|
|
||||||
BooleanQuery.setMaxClauseCount(cfg.getInt("index", "maxTerms",
|
|
||||||
BooleanQuery.getMaxClauseCount()));
|
|
||||||
|
|
||||||
GerritIndexWriterConfig openConfig =
|
GerritIndexWriterConfig openConfig =
|
||||||
new GerritIndexWriterConfig(cfg, "changes_open");
|
new GerritIndexWriterConfig(cfg, "changes_open");
|
||||||
GerritIndexWriterConfig closedConfig =
|
GerritIndexWriterConfig closedConfig =
|
||||||
new GerritIndexWriterConfig(cfg, "changes_closed");
|
new GerritIndexWriterConfig(cfg, "changes_closed");
|
||||||
|
|
||||||
|
queryBuilder = new QueryBuilder(openConfig.getAnalyzer());
|
||||||
|
|
||||||
|
BooleanQuery.setMaxClauseCount(cfg.getInt("index", "maxTerms",
|
||||||
|
BooleanQuery.getMaxClauseCount()));
|
||||||
|
|
||||||
SearcherFactory searcherFactory = new SearcherFactory();
|
SearcherFactory searcherFactory = new SearcherFactory();
|
||||||
if (cfg.getBoolean("index", "lucene", "testInmemory", false)) {
|
if (cfg.getBoolean("index", "lucene", "testInmemory", false)) {
|
||||||
openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig,
|
openIndex = new SubIndex(new RAMDirectory(), "ramOpen", openConfig,
|
||||||
|
@@ -20,7 +20,6 @@ import com.google.common.collect.Sets;
|
|||||||
import com.google.common.util.concurrent.AbstractFuture;
|
import com.google.common.util.concurrent.AbstractFuture;
|
||||||
import com.google.common.util.concurrent.ListenableFuture;
|
import com.google.common.util.concurrent.ListenableFuture;
|
||||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||||
import com.google.gerrit.lucene.LuceneChangeIndex.GerritIndexWriterConfig;
|
|
||||||
|
|
||||||
import org.apache.lucene.document.Document;
|
import org.apache.lucene.document.Document;
|
||||||
import org.apache.lucene.index.IndexWriter;
|
import org.apache.lucene.index.IndexWriter;
|
||||||
|
Reference in New Issue
Block a user