opendev/gerrit
Code Issues Proposed changes

Move Document generation to AbstractLuceneIndex

Browse Source 
Change-Id: If556022560553450b08e097c7d6e3a7644528c17
This commit is contained in:
Dave Borowitz
2016-03-16 14:34:23 +01:00
parent aa01c29b5b
commit cbb99428d8
3 changed files with 98 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
gerrit-lucene/src/main/java/com/google/gerrit/lucene/AbstractLuceneIndex.java
View File

@@ -21,10 +21,21 @@ import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.Schema.Values;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TrackingIndexWriter;
@@ -41,6 +52,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
@@ -53,6 +65,10 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
private static final Logger log =
LoggerFactory.getLogger(AbstractLuceneIndex.class);
static String sortFieldName(FieldDef<?, ?> f) {
return f.getName() + "_SORT";
}
public static void setReady(SitePaths sitePaths, int version, boolean ready)
throws IOException {
try {
@@ -228,6 +244,55 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
searcherManager.release(searcher);
}
Document toDocument(V obj, FillArgs fillArgs) {
Document result = new Document();
for (Values<V> vs : schema.buildFields(obj, fillArgs)) {
if (vs.getValues() != null) {
add(result, vs);
}
}
return result;
}
void add(Document doc, Values<V> values) {
String name = values.getField().getName();
FieldType<?> type = values.getField().getType();
Store store = store(values.getField());
if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) {
for (Object value : values.getValues()) {
doc.add(new IntField(name, (Integer) value, store));
}
} else if (type == FieldType.LONG) {
for (Object value : values.getValues()) {
doc.add(new LongField(name, (Long) value, store));
}
} else if (type == FieldType.TIMESTAMP) {
for (Object value : values.getValues()) {
doc.add(new LongField(name, ((Timestamp) value).getTime(), store));
}
} else if (type == FieldType.EXACT
|| type == FieldType.PREFIX) {
for (Object value : values.getValues()) {
doc.add(new StringField(name, (String) value, store));
}
} else if (type == FieldType.FULL_TEXT) {
for (Object value : values.getValues()) {
doc.add(new TextField(name, (String) value, store));
}
} else if (type == FieldType.STORED_ONLY) {
for (Object value : values.getValues()) {
doc.add(new StoredField(name, (byte[]) value));
}
} else {
throw FieldType.badFieldType(type);
}
}
private static Field.Store store(FieldDef<?, ?> f) {
return f.isStored() ? Field.Store.YES : Field.Store.NO;
}
private final class NrtFuture extends AbstractFuture<Void> {
private final long gen;

24
gerrit-lucene/src/main/java/com/google/gerrit/lucene/ChangeSubIndex.java
View File

@@ -14,22 +14,32 @@
package com.google.gerrit.lucene;
import static com.google.common.collect.Iterables.getOnlyElement;
import static com.google.gerrit.lucene.LuceneChangeIndex.ID_SORT_FIELD;
import static com.google.gerrit.lucene.LuceneChangeIndex.UPDATED_SORT_FIELD;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.Schema.Values;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeIndex;
import com.google.gerrit.server.query.DataSource;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import java.io.IOException;
import java.nio.file.Path;
import java.sql.Timestamp;
public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData>
implements ChangeIndex {
@@ -71,4 +81,18 @@ public class ChangeSubIndex extends AbstractLuceneIndex<Change.Id, ChangeData>
throw new UnsupportedOperationException(
"don't use ChangeSubIndex directly");
}
@Override
void add(Document doc, Values<ChangeData> values) {
// Add separate DocValues fields for those fields needed for sorting.
FieldDef<ChangeData, ?> f = values.getField();
if (f == ChangeField.LEGACY_ID) {
int v = (Integer) getOnlyElement(values.getValues());
doc.add(new NumericDocValuesField(ID_SORT_FIELD, v));
} else if (f == ChangeField.UPDATED) {
long t = ((Timestamp) getOnlyElement(values.getValues())).getTime();
doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t));
}
super.add(doc, values);
}
}

91
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@@ -15,7 +15,7 @@
package com.google.gerrit.lucene;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.getOnlyElement;
import static com.google.gerrit.lucene.AbstractLuceneIndex.sortFieldName;
import static com.google.gerrit.lucene.LuceneVersionManager.CHANGES_PREFIX;
import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE;
import static com.google.gerrit.server.index.change.ChangeField.CHANGE;
@@ -37,13 +37,10 @@ import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.QueryOptions;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.Schema.Values;
import com.google.gerrit.server.index.change.ChangeField;
import com.google.gerrit.server.index.change.ChangeField.ChangeProtoField;
import com.google.gerrit.server.index.change.ChangeField.PatchSetApprovalProtoField;
@@ -62,14 +59,6 @@ import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
@@ -90,7 +79,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Path;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
@@ -113,6 +101,11 @@ public class LuceneChangeIndex implements ChangeIndex {
public static final String CHANGES_OPEN = "open";
public static final String CHANGES_CLOSED = "closed";
static final String UPDATED_SORT_FIELD =
sortFieldName(ChangeField.UPDATED);
static final String ID_SORT_FIELD =
sortFieldName(ChangeField.LEGACY_ID);
private static final String ADDED_FIELD = ChangeField.ADDED.getName();
private static final String APPROVAL_FIELD = ChangeField.APPROVAL.getName();
private static final String CHANGE_FIELD = ChangeField.CHANGE.getName();
@@ -121,10 +114,6 @@ public class LuceneChangeIndex implements ChangeIndex {
private static final String PATCH_SET_FIELD = ChangeField.PATCH_SET.getName();
private static final String REVIEWEDBY_FIELD =
ChangeField.REVIEWEDBY.getName();
private static final String UPDATED_SORT_FIELD =
sortFieldName(ChangeField.UPDATED);
private static final String ID_SORT_FIELD =
sortFieldName(ChangeField.LEGACY_ID);
static Term idTerm(ChangeData cd) {
return QueryBuilder.intTerm(LEGACY_ID.getName(), cd.getId().get());
@@ -134,10 +123,6 @@ public class LuceneChangeIndex implements ChangeIndex {
return QueryBuilder.intTerm(LEGACY_ID.getName(), id.get());
}
private static String sortFieldName(FieldDef<?, ?> f) {
return f.getName() + "_SORT";
}
static interface Factory {
LuceneChangeIndex create(Schema<ChangeData> schema);
}
@@ -216,7 +201,9 @@ public class LuceneChangeIndex implements ChangeIndex {
@Override
public void replace(ChangeData cd) throws IOException {
Term id = LuceneChangeIndex.idTerm(cd);
Document doc = toDocument(cd);
// toDocument is essentially static and doesn't depend on the specific
// sub-index, so just pick one.
Document doc = openIndex.toDocument(cd, fillArgs);
try {
if (cd.change().getStatus().isOpen()) {
Futures.allAsList(
@@ -495,64 +482,4 @@ public class LuceneChangeIndex implements ChangeIndex {
}
return result;
}
private Document toDocument(ChangeData cd) {
Document result = new Document();
for (Values<ChangeData> vs : schema.buildFields(cd, fillArgs)) {
if (vs.getValues() != null) {
add(result, vs);
}
}
return result;
}
private static <V> void add(Document doc, Values<V> values) {
String name = values.getField().getName();
FieldType<?> type = values.getField().getType();
Store store = store(values.getField());
FieldDef<V, ?> f = values.getField();
// Add separate DocValues fields for those fields needed for sorting.
if (f == ChangeField.LEGACY_ID) {
int v = (Integer) getOnlyElement(values.getValues());
doc.add(new NumericDocValuesField(sortFieldName(f), v));
} else if (f == ChangeField.UPDATED) {
long t = ((Timestamp) getOnlyElement(values.getValues())).getTime();
doc.add(new NumericDocValuesField(UPDATED_SORT_FIELD, t));
}
if (type == FieldType.INTEGER || type == FieldType.INTEGER_RANGE) {
for (Object value : values.getValues()) {
doc.add(new IntField(name, (Integer) value, store));
}
} else if (type == FieldType.LONG) {
for (Object value : values.getValues()) {
doc.add(new LongField(name, (Long) value, store));
}
} else if (type == FieldType.TIMESTAMP) {
for (Object value : values.getValues()) {
doc.add(new LongField(name, ((Timestamp) value).getTime(), store));
}
} else if (type == FieldType.EXACT
|| type == FieldType.PREFIX) {
for (Object value : values.getValues()) {
doc.add(new StringField(name, (String) value, store));
}
} else if (type == FieldType.FULL_TEXT) {
for (Object value : values.getValues()) {
doc.add(new TextField(name, (String) value, store));
}
} else if (type == FieldType.STORED_ONLY) {
for (Object value : values.getValues()) {
doc.add(new StoredField(name, (byte[]) value));
}
} else {
throw FieldType.badFieldType(type);
}
}
private static Field.Store store(FieldDef<?, ?> f) {
return f.isStored() ? Field.Store.YES : Field.Store.NO;
}
}