opendev/gerrit
Code Issues Proposed changes

Render JSON results for the change table only using the index

Browse Source 
Define a new type of field, STORED_ONLY, for storing arbitrary raw
bytes that cannot be searched. Add two fields of this type storing the
Change object and the list of SubmitRecord.Labels, which is exactly
the set of data needed to render a ChangeTable2. This avoids hitting
the database in the common case of rendering dashboards and search
results in the web UI, which may be costly on non-SQL implementations
Similarly we can avoid running and re-running the submit rule
evaluator.

This implementation has a minor bug in that changes to submit rules
may invalidate previously indexed labels, as can changing the set of
plugins with PredicateProviders.

There is substantial collateral damage to Reindex stemming from the
fact that indexing now depends on the submit rule evaluator, which in
turn requires pulling in plugins and hence a lot more Guice glue.

Change-Id: I3c10ed936eb9258e453c46f947abf35f5b8fb308
This commit is contained in:
Dave Borowitz
2013-09-10 15:23:57 -07:00
parent 2474f211d4
commit 05b254feb2
10 changed files with 373 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@@ -29,6 +29,8 @@ import com.google.gerrit.reviewdb.client.SubmitRecord;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.ChangeField.ChangeProtoField;
import com.google.gerrit.server.index.ChangeField.SubmitLabelProtoField;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
@@ -51,6 +53,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
@@ -64,6 +67,7 @@ import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
@@ -99,6 +103,9 @@ public class LuceneChangeIndex implements ChangeIndex {
public static final String CHANGES_OPEN = "open";
public static final String CHANGES_CLOSED = "closed";
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
private static final String CHANGE_FIELD = ChangeField.CHANGE.getName();
private static final String SUBMIT_LABEL_FIELD =
ChangeField.SUBMIT_RECORD_LABEL.getName();
static interface Factory {
LuceneChangeIndex create(Schema<ChangeData> schema, String base);
@@ -257,7 +264,8 @@ public class LuceneChangeIndex implements ChangeIndex {
}
private static class QuerySource implements ChangeDataSource {
private static final ImmutableSet<String> FIELDS = ImmutableSet.of(ID_FIELD);
private static final ImmutableSet<String> FIELDS =
ImmutableSet.of(ID_FIELD, CHANGE_FIELD, SUBMIT_LABEL_FIELD);
private final List<SubIndex> indexes;
private final Query query;
@@ -304,8 +312,7 @@ public class LuceneChangeIndex implements ChangeIndex {
Lists.newArrayListWithCapacity(docs.scoreDocs.length);
for (ScoreDoc sd : docs.scoreDocs) {
Document doc = searchers[sd.shardIndex].doc(sd.doc, FIELDS);
Number v = doc.getField(ID_FIELD).numericValue();
result.add(new ChangeData(new Change.Id(v.intValue())));
result.add(toChangeData(doc));
}
final List<ChangeData> r = Collections.unmodifiableList(result);
@@ -341,6 +348,30 @@ public class LuceneChangeIndex implements ChangeIndex {
}
}
private static ChangeData toChangeData(Document doc) {
BytesRef cb = doc.getBinaryValue(CHANGE_FIELD);
if (cb == null) {
int id = doc.getField(ID_FIELD).numericValue().intValue();
return new ChangeData(new Change.Id(id));
}
Change change = ChangeProtoField.CODEC.decode(
cb.bytes, cb.offset, cb.length);
ChangeData cd = new ChangeData(change);
BytesRef[] labelsBytes = doc.getBinaryValues(SUBMIT_LABEL_FIELD);
if (labelsBytes != null) {
List<SubmitRecord.Label> labels =
Lists.newArrayListWithCapacity(labelsBytes.length);
for (BytesRef lb : labelsBytes) {
labels.add(SubmitLabelProtoField.CODEC.decode(
lb.bytes, lb.offset, lb.length));
}
cd.setSubmitRecordLabels(labels);
}
return cd;
}
private Document toDocument(ChangeData cd) throws IOException {
try {
Document result = new Document();
@@ -387,6 +418,10 @@ public class LuceneChangeIndex implements ChangeIndex {
for (Object value : values) {
doc.add(new TextField(name, (String) value, store));
}
} else if (f.getType() == FieldType.STORED_ONLY) {
for (Object value : values) {
doc.add(new StoredField(name, (byte[]) value));
}
} else {
throw QueryBuilder.badFieldType(f.getType());
}