opendev/gerrit
Code Issues Proposed changes

Search age (aka last updated) with ChangeIndex

Browse Source 
The age operator is implemented as a range query on the updated
ChangeField.  To save space the value is represented as the number
of minutes since the epoch, removing about 16 bits from the 64 bit
timestamp.  The field is stored to permit a future change to sort.

Aside from age also send sortkey_before and sortkey_after to the
ChangeIndex.  These drive off the same updated field as age, and
round out the set of operators that support range queries.

With this complete the ChangeIndex can answer many common user
queries without hitting the database. Result sorting still loads
the change records, to be fixed in a future commit.

Change-Id: Ib8056f21d866904122d37241cc52f69a998b30c2
This commit is contained in:
Shawn Pearce
2013-06-25 15:06:38 -06:00
committed by Dave Borowitz
parent 6a7dde6d95
commit 95b77ffdf3
9 changed files with 170 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@@ -34,6 +34,7 @@ import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexPredicate;
import com.google.gerrit.server.index.TimestampRangePredicate;
import com.google.gerrit.server.query.AndPredicate;
import com.google.gerrit.server.query.NotPredicate;
import com.google.gerrit.server.query.OrPredicate;
@@ -48,6 +49,7 @@ import com.google.gwtorm.server.ResultSet;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
@@ -57,6 +59,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherManager;
@@ -71,6 +74,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -221,6 +225,10 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
} else if (p.getClass() == OrPredicate.class) {
return booleanQuery(p, SHOULD);
} else if (p.getClass() == NotPredicate.class) {
if (p.getChild(0) instanceof TimestampRangePredicate) {
return notTimestampQuery(
(TimestampRangePredicate<ChangeData>) p.getChild(0));
}
return booleanQuery(p, MUST_NOT);
} else if (p instanceof IndexPredicate) {
return fieldQuery((IndexPredicate<ChangeData>) p);
@@ -242,6 +250,8 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
throws QueryParseException {
if (p.getType() == FieldType.INTEGER) {
return intQuery(p);
} else if (p.getType() == FieldType.TIMESTAMP) {
return timestampQuery(p);
} else if (p.getType() == FieldType.EXACT) {
return exactQuery(p);
} else {
@@ -268,6 +278,32 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
return new TermQuery(intTerm(p.getOperator(), value));
}
private static Query timestampQuery(IndexPredicate<ChangeData> p)
throws QueryParseException {
if (p instanceof TimestampRangePredicate) {
TimestampRangePredicate<ChangeData> r =
(TimestampRangePredicate<ChangeData>) p;
return NumericRangeQuery.newIntRange(
r.getField().getName(),
toIndexTime(r.getMinTimestamp()),
toIndexTime(r.getMaxTimestamp()),
true, true);
}
throw new QueryParseException("not a timestamp: " + p);
}
private static Query notTimestampQuery(TimestampRangePredicate<ChangeData> r)
throws QueryParseException {
if (r.getMinTimestamp().getTime() == 0) {
return NumericRangeQuery.newIntRange(
r.getField().getName(),
toIndexTime(r.getMaxTimestamp()),
Integer.MAX_VALUE,
true, true);
}
throw new QueryParseException("cannot negate: " + r);
}
private Query exactQuery(IndexPredicate<ChangeData> p) {
return new TermQuery(new Term(p.getOperator(), p.getValue()));
}
@@ -367,19 +403,30 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
private void add(Document doc, FieldDef<ChangeData, ?> f,
Iterable<?> values) throws OrmException {
String name = f.getName();
Store store = store(f);
if (f.getType() == FieldType.INTEGER) {
for (Object value : values) {
doc.add(new IntField(f.getName(), (Integer) value, store(f)));
doc.add(new IntField(name, (Integer) value, store));
}
} else if (f.getType() == FieldType.TIMESTAMP) {
for (Object v : values) {
doc.add(new IntField(name, toIndexTime((Timestamp) v), store));
}
} else if (f.getType() == FieldType.EXACT) {
for (Object value : values) {
doc.add(new StringField(f.getName(), (String) value, store(f)));
doc.add(new StringField(name, (String) value, store));
}
} else {
throw badFieldType(f.getType());
}
}
private static int toIndexTime(Timestamp ts) {
return (int) (ts.getTime() / 60000);
}
private static Field.Store store(FieldDef<?, ?> f) {
return f.isStored() ? Field.Store.YES : Field.Store.NO;
}

15
gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java
View File

@@ -75,6 +75,7 @@ import java.util.Set;
import java.util.regex.Matcher;
public class ChangeUtil {
private static final long SORT_KEY_EPOCH = 1222819200L; // Oct 1 2008 00:00
private static final Object uuidLock = new Object();
private static final int SEED = 0x2418e6f9;
private static int uuidPrefix;
@@ -441,7 +442,7 @@ public class ChangeUtil {
// The encoding uses minutes since Wed Oct 1 00:00:00 2008 UTC.
// We overrun approximately 4,085 years later, so ~6093.
//
final long lastUpdatedOn = (lastUpdated / 1000L) - 1222819200L;
final long lastUpdatedOn = (lastUpdated / 1000L) - SORT_KEY_EPOCH;
final StringBuilder r = new StringBuilder(16);
r.setLength(16);
formatHexInt(r, 0, (int) (lastUpdatedOn / 60));
@@ -449,6 +450,18 @@ public class ChangeUtil {
return r.toString();
}
public static Timestamp timeFromSortKey(String sortKey) {
if ("z".equals(sortKey)) {
return new Timestamp(Long.MAX_VALUE);
}
String ts = sortKey.substring(0, 8);
int i = 0;
while (i < 8 && ts.charAt(i) == '0')
i++;
long v = Long.parseLong(ts.substring(i), 16) * 60;
return new Timestamp((v + SORT_KEY_EPOCH) * 1000);
}
public static void computeSortKey(final Change c) {
long lastUpdated = c.getLastUpdatedOn().getTime();
int id = c.getId().get();

5
gerrit-server/src/main/java/com/google/gerrit/server/change/PostReview.java
View File

@@ -42,6 +42,7 @@ import com.google.gerrit.server.ChangeUtil;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.account.AccountsCollection;
import com.google.gerrit.server.change.PostReview.Input;
import com.google.gerrit.server.index.ChangeIndexer;
import com.google.gerrit.server.project.ChangeControl;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Inject;
@@ -118,6 +119,7 @@ public class PostReview implements RestModifyView<RevisionResource, Input> {
}
private final ReviewDb db;
private final ChangeIndexer indexer;
private final AccountsCollection accounts;
private final EmailReviewComments.Factory email;
@Deprecated private final ChangeHooks hooks;
@@ -131,10 +133,12 @@ public class PostReview implements RestModifyView<RevisionResource, Input> {
@Inject
PostReview(ReviewDb db,
ChangeIndexer indexer,
AccountsCollection accounts,
EmailReviewComments.Factory email,
ChangeHooks hooks) {
this.db = db;
this.indexer = indexer;
this.accounts = accounts;
this.email = email;
this.hooks = hooks;
@@ -171,6 +175,7 @@ public class PostReview implements RestModifyView<RevisionResource, Input> {
if (dirty) {
db.changes().update(Collections.singleton(change));
db.commit();
indexer.index(change);
}
} finally {
db.rollback();

14
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java
View File

@@ -24,6 +24,7 @@ import com.google.gwtorm.server.OrmException;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.sql.Timestamp;
import java.util.Map;
/**
@@ -38,7 +39,7 @@ import java.util.Map;
*/
public class ChangeField {
/** Increment whenever making schema changes. */
public static final int SCHEMA_VERSION = 2;
public static final int SCHEMA_VERSION = 3;
/** Legacy change ID. */
public static final FieldDef<ChangeData, Integer> CHANGE_ID =
@@ -95,6 +96,17 @@ public class ChangeField {
}
};
/** Last update time since January 1, 1970. */
public static final FieldDef<ChangeData, Timestamp> UPDATED =
new FieldDef.Single<ChangeData, Timestamp>(
"updated", FieldType.TIMESTAMP, true) {
@Override
public Timestamp get(ChangeData input, FillArgs args)
throws OrmException {
return input.change(args.db).getLastUpdatedOn();
}
};
/** List of filenames modified in the current patch set. */
public static final FieldDef<ChangeData, Iterable<String>> FILE =
new FieldDef.Repeatable<ChangeData, String>(

6
gerrit-server/src/main/java/com/google/gerrit/server/index/FieldType.java
View File

@@ -14,6 +14,8 @@
package com.google.gerrit.server.index;
import java.sql.Timestamp;
/** Document field types supported by the secondary index system. */
public class FieldType<T> {
@@ -21,6 +23,10 @@ public class FieldType<T> {
public static final FieldType<Integer> INTEGER =
new FieldType<Integer>("INTEGER");
/** A single date/time-valued field. */
public static final FieldType<Timestamp> TIMESTAMP =
new FieldType<Timestamp>("TIMESTAMP");
/** A string field searched using exact-match semantics. */
public static final FieldType<String> EXACT =
new FieldType<String>("EXACT");

9
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexPredicate.java
View File

@@ -25,6 +25,15 @@ public abstract class IndexPredicate<I> extends OperatorPredicate<I> {
this.def = def;
}
protected IndexPredicate(FieldDef<I, ?> def, String name, String value) {
super(name, value);
this.def = def;
}
public FieldDef<I, ?> getField() {
return def;
}
public FieldType<?> getType() {
return def.getType();
}

27
gerrit-server/src/main/java/com/google/gerrit/server/index/TimestampRangePredicate.java Normal file
View File

@@ -0,0 +1,27 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.package com.google.gerrit.server.git;
package com.google.gerrit.server.index;
import java.sql.Timestamp;
public abstract class TimestampRangePredicate<I> extends IndexPredicate<I> {
protected TimestampRangePredicate(FieldDef<I, Timestamp> def,
String name, String value) {
super(def, name, value);
}
public abstract Timestamp getMinTimestamp();
public abstract Timestamp getMaxTimestamp();
}

23
gerrit-server/src/main/java/com/google/gerrit/server/query/change/AgePredicate.java
View File

@@ -20,31 +20,42 @@ import static java.util.concurrent.TimeUnit.SECONDS;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.config.ConfigUtil;
import com.google.gerrit.server.query.OperatorPredicate;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.TimestampRangePredicate;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Provider;
class AgePredicate extends OperatorPredicate<ChangeData> {
import java.sql.Timestamp;
public class AgePredicate extends TimestampRangePredicate<ChangeData> {
private final Provider<ReviewDb> dbProvider;
private final long cut;
AgePredicate(Provider<ReviewDb> dbProvider, String value) {
super(ChangeQueryBuilder.FIELD_AGE, value);
super(ChangeField.UPDATED, ChangeQueryBuilder.FIELD_AGE, value);
this.dbProvider = dbProvider;
long s = ConfigUtil.getTimeUnit(getValue(), 0, SECONDS);
long ms = MILLISECONDS.convert(s, SECONDS);
this.cut = (System.currentTimeMillis() - ms) + 1;
this.cut = System.currentTimeMillis() - ms;
}
public Timestamp getMinTimestamp() {
return new Timestamp(0);
}
public Timestamp getMaxTimestamp() {
return new Timestamp(cut);
}
long getCut() {
return cut;
return cut + 1;
}
@Override
public boolean match(final ChangeData object) throws OrmException {
Change change = object.change(dbProvider);
return change != null && change.getLastUpdatedOn().getTime() < cut;
return change != null && change.getLastUpdatedOn().getTime() <= cut;
}
@Override

35
gerrit-server/src/main/java/com/google/gerrit/server/query/change/SortKeyPredicate.java
View File

@@ -16,15 +16,20 @@ package com.google.gerrit.server.query.change;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.query.OperatorPredicate;
import com.google.gerrit.server.ChangeUtil;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.TimestampRangePredicate;
import com.google.gwtorm.server.OrmException;
import com.google.inject.Provider;
abstract class SortKeyPredicate extends OperatorPredicate<ChangeData> {
import java.sql.Timestamp;
public abstract class SortKeyPredicate extends
TimestampRangePredicate<ChangeData> {
protected final Provider<ReviewDb> dbProvider;
SortKeyPredicate(Provider<ReviewDb> dbProvider, String name, String value) {
super(name, value);
super(ChangeField.UPDATED, name, value);
this.dbProvider = dbProvider;
}
@@ -33,11 +38,21 @@ abstract class SortKeyPredicate extends OperatorPredicate<ChangeData> {
return 1;
}
static class Before extends SortKeyPredicate {
public static class Before extends SortKeyPredicate {
Before(Provider<ReviewDb> dbProvider, String value) {
super(dbProvider, "sortkey_before", value);
}
@Override
public Timestamp getMinTimestamp() {
return new Timestamp(0);
}
@Override
public Timestamp getMaxTimestamp() {
return ChangeUtil.timeFromSortKey(getValue());
}
@Override
public boolean match(ChangeData cd) throws OrmException {
Change change = cd.change(dbProvider);
@@ -45,11 +60,21 @@ abstract class SortKeyPredicate extends OperatorPredicate<ChangeData> {
}
}
static class After extends SortKeyPredicate {
public static class After extends SortKeyPredicate {
After(Provider<ReviewDb> dbProvider, String value) {
super(dbProvider, "sortkey_after", value);
}
@Override
public Timestamp getMinTimestamp() {
return ChangeUtil.timeFromSortKey(getValue());
}
@Override
public Timestamp getMaxTimestamp() {
return new Timestamp(Long.MAX_VALUE);
}
@Override
public boolean match(ChangeData cd) throws OrmException {
Change change = cd.change(dbProvider);