opendev/gerrit
Code Issues Proposed changes

Support pagination using offsets instead of sortkey

Browse Source 
We cannot guarantee that secondary index implementations (particularly
the one used by googlesource.com) can efficiently paginate based on
the sortkey, and in particular can both sort and reverse sort on the
same field. This particular performance issue notwithstanding,
searching is now generally fast enough that it is feasible just to
skip the first N results when doing pagination.

Add an option S= to QueryChanges to support starting at a nonzero
offset. Note that we still have to fetch n+S results from the index in
order to do visibility filtering, since if we skipped at the index
layer we wouldn't know how many of the skipped elements would have
matched later filtering.

Drop the sortkey token suffix from the legacy anchor parser; there is
no reliable way to convert it to an offset, and it's unlikely that
users have permalinks to specific sortkey values.

On the server side, remove the sortkey field from the current index
version, and use pagination by offset instead of sortkey in the new
version only.

Continue to support sortkey queries against old index versions, to
support online reindexing while clients have an older JS version.

Change-Id: I6a82965db02c4d534e2107ca6ec91217085124d6
This commit is contained in:
Dave Borowitz
2014-02-10 15:58:20 -08:00
parent a0af7febe6
commit 86caf9ec23
40 changed files with 645 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@@ -48,6 +48,7 @@ import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gerrit.server.query.change.ChangeQueryBuilder;
import com.google.gerrit.server.query.change.SortKeyPredicate;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Provider;
@@ -300,8 +301,8 @@ public class LuceneChangeIndex implements ChangeIndex {
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p, int limit)
throws QueryParseException {
public ChangeDataSource getSource(Predicate<ChangeData> p, int start,
int limit) throws QueryParseException {
Set<Change.Status> statuses = IndexRewriteImpl.getPossibleStatus(p);
List<SubIndex> indexes = Lists.newArrayListWithCapacity(2);
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
@@ -310,8 +311,8 @@ public class LuceneChangeIndex implements ChangeIndex {
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(closedIndex);
}
return new QuerySource(indexes, queryBuilder.toQuery(p), limit,
ChangeQueryBuilder.hasNonTrivialSortKeyAfter(schema, p));
return new QuerySource(indexes, queryBuilder.toQuery(p), start, limit,
getSort(schema, p));
}
@Override
@@ -322,18 +323,38 @@ public class LuceneChangeIndex implements ChangeIndex {
private static final ImmutableSet<String> FIELDS =
ImmutableSet.of(ID_FIELD, CHANGE_FIELD, APPROVAL_FIELD);
@SuppressWarnings("deprecation")
private static Sort getSort(Schema<ChangeData> schema,
Predicate<ChangeData> p) {
// Standard order is descending by sort key, unless reversed due to a
// sortkey_before predicate.
if (SortKeyPredicate.hasSortKeyField(schema)) {
boolean reverse = ChangeQueryBuilder.hasNonTrivialSortKeyAfter(schema, p);
return new Sort(new SortField(
ChangeField.SORTKEY.getName(), SortField.Type.LONG, !reverse));
} else {
return new Sort(
new SortField(
ChangeField.UPDATED.getName(), SortField.Type.LONG, true),
new SortField(
ChangeField.LEGACY_ID.getName(), SortField.Type.INT, true));
}
}
private class QuerySource implements ChangeDataSource {
private final List<SubIndex> indexes;
private final Query query;
private final int start;
private final int limit;
private final boolean reverse;
private final Sort sort;
private QuerySource(List<SubIndex> indexes, Query query, int limit,
boolean reverse) {
private QuerySource(List<SubIndex> indexes, Query query, int start,
int limit, Sort sort) {
this.indexes = indexes;
this.query = query;
this.start = start;
this.limit = limit;
this.reverse = reverse;
this.sort = sort;
}
@Override
@@ -354,24 +375,19 @@ public class LuceneChangeIndex implements ChangeIndex {
@Override
public ResultSet<ChangeData> read() throws OrmException {
IndexSearcher[] searchers = new IndexSearcher[indexes.size()];
Sort sort = new Sort(
new SortField(
ChangeField.SORTKEY.getName(),
SortField.Type.LONG,
// Standard order is descending by sort key, unless reversed due
// to a sortkey_before predicate.
!reverse));
try {
int realLimit = start + limit;
TopDocs[] hits = new TopDocs[indexes.size()];
for (int i = 0; i < indexes.size(); i++) {
searchers[i] = indexes.get(i).acquire();
hits[i] = searchers[i].search(query, limit, sort);
hits[i] = searchers[i].search(query, realLimit, sort);
}
TopDocs docs = TopDocs.merge(sort, limit, hits);
TopDocs docs = TopDocs.merge(sort, realLimit, hits);
List<ChangeData> result =
Lists.newArrayListWithCapacity(docs.scoreDocs.length);
for (ScoreDoc sd : docs.scoreDocs) {
for (int i = start; i < docs.scoreDocs.length; i++) {
ScoreDoc sd = docs.scoreDocs[i];
Document doc = searchers[sd.shardIndex].doc(sd.doc, FIELDS);
result.add(toChangeData(doc));
}
@@ -462,9 +478,17 @@ public class LuceneChangeIndex implements ChangeIndex {
doc.add(new LongField(name, (Long) value, store));
}
} else if (type == FieldType.TIMESTAMP) {
for (Object value : values.getValues()) {
int t = QueryBuilder.toIndexTime((Timestamp) value);
doc.add(new IntField(name, t, store));
@SuppressWarnings("deprecation")
boolean legacy = values.getField() == ChangeField.LEGACY_UPDATED;
if (legacy) {
for (Object value : values.getValues()) {
int t = queryBuilder.toIndexTimeInMinutes((Timestamp) value);
doc.add(new IntField(name, (int) t, store));
}
} else {
for (Object value : values.getValues()) {
doc.add(new LongField(name, ((Timestamp) value).getTime(), store));
}
}
} else if (type == FieldType.EXACT
|| type == FieldType.PREFIX) {