opendev/gerrit
Code Issues Proposed changes

Update Lucene to 4.4.0

Browse Source 
Of note in this version upgrade is the replacement of NrtManager by
ControlledRealTimeReopenThread, which has a different API as used by
SubIndex.

Change-Id: Id34eb8e75bb07a38be9b5ed9a1cdeaf195774b58
This commit is contained in:
Dave Borowitz 2013-09-30 17:05:23 -07:00
parent d57c22d4ee
commit 3d271c0c66
5 changed files with 93 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@ -14,15 +14,18 @@
package com.google.gerrit.lucene;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.gerrit.server.index.IndexRewriteImpl.CLOSED_STATUSES;
import static com.google.gerrit.server.index.IndexRewriteImpl.OPEN_STATUSES;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.reviewdb.client.Change;
@ -33,6 +36,7 @@ import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.ChangeField.ChangeProtoField;
import com.google.gerrit.server.index.ChangeField.PatchSetApprovalProtoField;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
@ -84,9 +88,8 @@ import java.sql.Timestamp;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
/**
* Secondary index implementation using Apache Lucene.
@ -100,20 +103,37 @@ public class LuceneChangeIndex implements ChangeIndex {
private static final Logger log =
LoggerFactory.getLogger(LuceneChangeIndex.class);
public static final Version LUCENE_VERSION = Version.LUCENE_43;
public static final String CHANGES_OPEN = "open";
public static final String CHANGES_CLOSED = "closed";
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
private static final String CHANGE_FIELD = ChangeField.CHANGE.getName();
private static final String APPROVAL_FIELD = ChangeField.APPROVAL.getName();
private static final Map<Schema<ChangeData>, Version> LUCENE_VERSIONS;
static {
ImmutableMap.Builder<Schema<ChangeData>, Version> versions =
ImmutableMap.builder();
@SuppressWarnings("deprecation")
Version lucene43 = Version.LUCENE_43;
for (Map.Entry<Integer, Schema<ChangeData>> e
: ChangeSchemas.ALL.entrySet()) {
if (e.getKey() <= 3) {
versions.put(e.getValue(), lucene43);
} else {
versions.put(e.getValue(), Version.LUCENE_44);
}
}
LUCENE_VERSIONS = versions.build();
}
static interface Factory {
LuceneChangeIndex create(Schema<ChangeData> schema, String base);
}
private static IndexWriterConfig getIndexWriterConfig(Config cfg, String name) {
IndexWriterConfig writerConfig = new IndexWriterConfig(LUCENE_VERSION,
new StandardAnalyzer(LUCENE_VERSION, CharArraySet.EMPTY_SET));
private static IndexWriterConfig getIndexWriterConfig(Version version,
Config cfg, String name) {
IndexWriterConfig writerConfig = new IndexWriterConfig(version,
new StandardAnalyzer(version, CharArraySet.EMPTY_SET));
writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
double m = 1 << 20;
writerConfig.setRAMBufferSizeMB(cfg.getLong("index", name, "ramBufferSize",
@ -125,7 +145,7 @@ public class LuceneChangeIndex implements ChangeIndex {
private final SitePaths sitePaths;
private final FillArgs fillArgs;
private final ExecutorService executor;
private final ListeningExecutorService executor;
private final File dir;
private final Schema<ChangeData> schema;
private final SubIndex openIndex;
@ -149,15 +169,18 @@ public class LuceneChangeIndex implements ChangeIndex {
} else {
dir = new File(base);
}
Version luceneVersion = checkNotNull(
LUCENE_VERSIONS.get(schema),
"unknown Lucene version for index schema: %s", schema);
openIndex = new SubIndex(new File(dir, CHANGES_OPEN),
getIndexWriterConfig(cfg, "changes_open"));
getIndexWriterConfig(luceneVersion, cfg, "changes_open"));
closedIndex = new SubIndex(new File(dir, CHANGES_CLOSED),
getIndexWriterConfig(cfg, "changes_closed"));
getIndexWriterConfig(luceneVersion, cfg, "changes_closed"));
}
@Override
public void close() {
List<Future<?>> closeFutures = Lists.newArrayListWithCapacity(2);
List<ListenableFuture<?>> closeFutures = Lists.newArrayListWithCapacity(2);
closeFutures.add(executor.submit(new Runnable() {
@Override
public void run() {
@ -170,9 +193,7 @@ public class LuceneChangeIndex implements ChangeIndex {
closedIndex.close();
}
}));
for (Future<?> future : closeFutures) {
Futures.getUnchecked(future);
}
Futures.getUnchecked(Futures.allAsList(closeFutures));
}
@Override

53
gerrit-lucene/src/main/java/com/google/gerrit/lucene/SubIndex.java
View File

@ -22,12 +22,12 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TrackingIndexWriter;
import org.apache.lucene.search.ControlledRealTimeReopenThread;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.NRTManager;
import org.apache.lucene.search.NRTManager.TrackingIndexWriter;
import org.apache.lucene.search.NRTManagerReopenThread;
import org.apache.lucene.search.ReferenceManager.RefreshListener;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.slf4j.Logger;
@ -48,17 +48,18 @@ class SubIndex {
private final Directory dir;
private final TrackingIndexWriter writer;
private final NRTManager nrtManager;
private final NRTManagerReopenThread reopenThread;
private final SearcherManager searcherManager;
private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
private final ConcurrentMap<RefreshListener, Boolean> refreshListeners;
SubIndex(File file, IndexWriterConfig writerConfig) throws IOException {
dir = FSDirectory.open(file);
writer = new NRTManager.TrackingIndexWriter(new IndexWriter(dir, writerConfig));
nrtManager = new NRTManager(writer, new SearcherFactory());
writer = new TrackingIndexWriter(new IndexWriter(dir, writerConfig));
searcherManager = new SearcherManager(
writer.getIndexWriter(), true, new SearcherFactory());
refreshListeners = Maps.newConcurrentMap();
nrtManager.addListener(new RefreshListener() {
searcherManager.addListener(new RefreshListener() {
@Override
public void beforeRefresh() throws IOException {
}
@ -71,8 +72,8 @@ class SubIndex {
}
});
reopenThread = new NRTManagerReopenThread(
nrtManager,
reopenThread = new ControlledRealTimeReopenThread<IndexSearcher>(
writer, searcherManager,
0.500 /* maximum stale age (seconds) */,
0.010 /* minimum stale age (seconds) */);
reopenThread.setName("NRT " + file.getName());
@ -86,12 +87,8 @@ class SubIndex {
void close() {
reopenThread.close();
try {
nrtManager.close();
} catch (IOException e) {
log.warn("error closing Lucene searcher", e);
}
try {
writer.getIndexWriter().close();
writer.getIndexWriter().commit();
writer.getIndexWriter().close(true);
} catch (IOException e) {
log.warn("error closing Lucene writer", e);
}
@ -119,11 +116,11 @@ class SubIndex {
}
IndexSearcher acquire() throws IOException {
return nrtManager.acquire();
return searcherManager.acquire();
}
void release(IndexSearcher searcher) throws IOException {
nrtManager.release(searcher);
searcherManager.release(searcher);
}
private final class NrtFuture extends AbstractFuture<Void>
@ -138,7 +135,7 @@ class SubIndex {
@Override
public Void get() throws InterruptedException, ExecutionException {
if (!isDone()) {
nrtManager.waitForGeneration(gen);
reopenThread.waitForGeneration(gen);
set(null);
}
return super.get();
@ -148,7 +145,8 @@ class SubIndex {
public Void get(long timeout, TimeUnit unit) throws InterruptedException,
TimeoutException, ExecutionException {
if (!isDone()) {
nrtManager.waitForGeneration(gen, timeout, unit);
reopenThread.waitForGeneration(gen,
(int) TimeUnit.MILLISECONDS.convert(timeout, unit));
set(null);
}
return super.get(timeout, unit);
@ -158,7 +156,7 @@ class SubIndex {
public boolean isDone() {
if (super.isDone()) {
return true;
} else if (gen <= nrtManager.getCurrentSearchingGen()) {
} else if (isSearcherCurrent()) {
set(null);
return true;
}
@ -168,7 +166,7 @@ class SubIndex {
@Override
public void addListener(Runnable listener, Executor executor) {
if (hasListeners.compareAndSet(false, true) && !isDone()) {
nrtManager.addListener(this);
searcherManager.addListener(this);
}
super.addListener(listener, executor);
}
@ -187,10 +185,19 @@ class SubIndex {
@Override
public void afterRefresh(boolean didRefresh) throws IOException {
if (gen <= nrtManager.getCurrentSearchingGen()) {
if (isSearcherCurrent()) {
refreshListeners.remove(this);
set(null);
}
}
private boolean isSearcherCurrent() {
try {
return reopenThread.waitForGeneration(gen, 0);
} catch (InterruptedException e) {
log.warn("Interrupted waiting for searcher generation", e);
return false;
}
}
}
}

10
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java
View File

@ -25,6 +25,7 @@ import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
/** Secondary index schemas for changes. */
@ -93,8 +94,15 @@ public class ChangeSchemas {
ChangeField.CHANGE,
ChangeField.APPROVAL);
// For upgrade to Lucene 4.4.0 index format only.
static final Schema<ChangeData> V4 = release(V3.getFields().values());
private static Schema<ChangeData> release(Collection<FieldDef<ChangeData, ?>> fields) {
return new Schema<ChangeData>(true, fields);
}
private static Schema<ChangeData> release(FieldDef<ChangeData, ?>... fields) {
return new Schema<ChangeData>(true, Arrays.asList(fields));
return release(Arrays.asList(fields));
}
@SuppressWarnings("unused")

8
gerrit-server/src/main/java/com/google/gerrit/server/index/Schema.java
View File

@ -15,6 +15,7 @@
package com.google.gerrit.server.index;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.collect.ImmutableMap;
/** Specific version of a secondary index schema. */
@ -51,6 +52,13 @@ public class Schema<T> {
return fields;
}
@Override
public String toString() {
return Objects.toStringHelper(this)
.addValue(fields.keySet())
.toString();
}
void setVersion(int version) {
this.version = version;
}

24
lib/lucene/BUCK
View File

@ -2,9 +2,9 @@ include_defs('//lib/maven.defs')
maven_jar(
name = 'core',
id = 'org.apache.lucene:lucene-core:4.3.0',
bin_sha1 = 'd4e40fe5661b8de5d8c66db3d63a47b6b3ecf7f3',
src_sha1 = '86c29288b1930e33ba7ffea1b866af9a52d3d24a',
id = 'org.apache.lucene:lucene-core:4.4.0',
bin_sha1 = 'a9a0b553d5f2444aea3340b22753ea4bbddaa0af',
src_sha1 = 'd321e15f688066a3c3598607303e0de452a076da',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE.txt',
@ -14,9 +14,9 @@ maven_jar(
maven_jar(
name = 'analyzers-common',
id = 'org.apache.lucene:lucene-analyzers-common:4.3.0',
bin_sha1 = 'e7c3976156d292f696016e138b67ab5e6bfc1a56',
src_sha1 = '3606622b3c1f09b4b7cf34070cbf60d414af9b6b',
id = 'org.apache.lucene:lucene-analyzers-common:4.4.0',
bin_sha1 = 'f58f6b727293b2d4392064db8c91fdf1d0eb4ffe',
src_sha1 = '60176bb63009f41104b42656b20c81b66313e7b5',
license = 'Apache2.0',
exclude = [
'META-INF/LICENSE.txt',
@ -26,17 +26,17 @@ maven_jar(
maven_jar(
name = 'highlighter',
id = 'org.apache.lucene:lucene-highlighter:4.3.0',
bin_sha1 = '9e6d60921e16a0d6b2e609c6a02a8b08cd7f643c',
src_sha1 = '0ff70cae1a8fb7af29bf254d90e9885961deed5e',
id = 'org.apache.lucene:lucene-highlighter:4.4.0',
bin_sha1 = 'c55f402683388c0a71a1dfaaff198873dfe5b1e4',
src_sha1 = '3a99f84e4b6a8f74c34b2f2bd076c9b2b46fff2e',
license = 'Apache2.0',
)
maven_jar(
name = 'queries',
id = 'org.apache.lucene:lucene-queries:4.3.0',
bin_sha1 = '68e01022bdf4f869b95362c9af964846e5d3cf2d',
src_sha1 = '3e3541c1b9f44c532ce88ab6a12216566c3399df',
id = 'org.apache.lucene:lucene-queries:4.4.0',
bin_sha1 = 'c9010f4852345ba2a65163fdeb17b7b653e4a3c4',
src_sha1 = 'eefbcd43e66747a412a9f186d183d187405374b8',
license = 'Apache2.0',
)