opendev/gerrit
Code Issues Proposed changes

Support pagination using offsets instead of sortkey

Browse Source 
We cannot guarantee that secondary index implementations (particularly
the one used by googlesource.com) can efficiently paginate based on
the sortkey, and in particular can both sort and reverse sort on the
same field. This particular performance issue notwithstanding,
searching is now generally fast enough that it is feasible just to
skip the first N results when doing pagination.

Add an option S= to QueryChanges to support starting at a nonzero
offset. Note that we still have to fetch n+S results from the index in
order to do visibility filtering, since if we skipped at the index
layer we wouldn't know how many of the skipped elements would have
matched later filtering.

Drop the sortkey token suffix from the legacy anchor parser; there is
no reliable way to convert it to an offset, and it's unlikely that
users have permalinks to specific sortkey values.

On the server side, remove the sortkey field from the current index
version, and use pagination by offset instead of sortkey in the new
version only.

Continue to support sortkey queries against old index versions, to
support online reindexing while clients have an older JS version.

Change-Id: I6a82965db02c4d534e2107ca6ec91217085124d6
This commit is contained in:
Dave Borowitz
2014-02-10 15:58:20 -08:00
parent a0af7febe6
commit 86caf9ec23
40 changed files with 645 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Documentation/config-gerrit.txt
View File

@@ -842,7 +842,7 @@ how the replacement is displayed to the user.
----
[commentlink "changeid"]
match = (I[0-9a-f]{8,40})
link = "#q,$1,n,z"
link = "#q,$1"
[commentlink "bugzilla"]
match = "(bug\\s+#?)(\\d+)"

2
Documentation/dev-contributing.txt
View File

@@ -27,7 +27,7 @@ review of your change, review these guidelines before submitting
your change. You can view the pending Gerrit contributions and
their statuses here:
* https://gerrit-review.googlesource.com/#/q/status:open+project:gerrit,n,z
* https://gerrit-review.googlesource.com/#/q/status:open+project:gerrit
Depending on the size of that list it might take a while for
your change to get reviewed. Naturally there are fewer

15
Documentation/user-search.txt
View File

@@ -469,21 +469,6 @@ automatically set to the page size configured in the current user's
preferences. Including it in a web query may lead to unpredictable
results with regards to pagination.
resume_sortkey:'KEY'::
+
Positions the low level scan routine to start from 'KEY' and
continue through changes from this point. This is most often used
for paginating result sets. Including this in a web query may lead
to unpredictable results.
sortkey_after:'KEY', sortkey_before:'KEY'::
+
Restart the low level scan routine from 'KEY'. This is automatically
set by the pagination system as the user navigates through results
of a query. Including either value in a web query may lead to
unpredictable results.
GERRIT
------
Part of link:index.html[Gerrit Code Review]

14
gerrit-common/src/main/java/com/google/gerrit/common/PageLinks.java
View File

@@ -35,9 +35,8 @@ public class PageLinks {
public static final String SETTINGS_NEW_AGREEMENT = "/settings/new-agreement";
public static final String REGISTER = "/register";
public static final String TOP = "n,z";
public static final String MINE = "/";
public static final String QUERY = "/q/";
public static final String PROJECTS = "/projects/";
public static final String DASHBOARDS = ",dashboards/";
public static final String ADMIN_GROUPS = "/admin/groups/";
@@ -84,7 +83,7 @@ public class PageLinks {
}
public static String toAccountQuery(String fullname, Status status) {
return toChangeQuery(op("owner", fullname) + " " + status(status), TOP);
return toChangeQuery(op("owner", fullname) + " " + status(status));
}
public static String toCustomDashboard(final String params) {
@@ -95,12 +94,13 @@ public class PageLinks {
return ADMIN_PROJECTS + proj.get() + ",dashboards";
}
public static String toChangeQuery(final String query) {
return toChangeQuery(query, TOP);
public static String toChangeQuery(String query) {
return QUERY + KeyUtil.encode(query);
}
public static String toChangeQuery(String query, String page) {
return "/q/" + KeyUtil.encode(query) + "," + page;
public static String toChangeQuery(String query, String start) {
int s = Integer.parseInt(start);
return QUERY + KeyUtil.encode(query) + (s > 0 ? "," + s : "");
}
public static String toProjectDashboard(Project.NameKey name, String id) {

54
gerrit-gwtui/src/main/java/com/google/gerrit/client/Dispatcher.java
View File

@@ -22,6 +22,7 @@ import static com.google.gerrit.common.PageLinks.ADMIN_PROJECTS;
import static com.google.gerrit.common.PageLinks.DASHBOARDS;
import static com.google.gerrit.common.PageLinks.MINE;
import static com.google.gerrit.common.PageLinks.PROJECTS;
import static com.google.gerrit.common.PageLinks.QUERY;
import static com.google.gerrit.common.PageLinks.REGISTER;
import static com.google.gerrit.common.PageLinks.SETTINGS;
import static com.google.gerrit.common.PageLinks.SETTINGS_AGREEMENTS;
@@ -34,6 +35,7 @@ import static com.google.gerrit.common.PageLinks.SETTINGS_PROJECTS;
import static com.google.gerrit.common.PageLinks.SETTINGS_SSHKEYS;
import static com.google.gerrit.common.PageLinks.SETTINGS_WEBIDENT;
import static com.google.gerrit.common.PageLinks.op;
import static com.google.gerrit.common.PageLinks.toChangeQuery;
import com.google.gerrit.client.account.MyAgreementsScreen;
import com.google.gerrit.client.account.MyContactInformationScreen;
@@ -83,8 +85,8 @@ import com.google.gerrit.client.ui.Screen;
import com.google.gerrit.common.PageLinks;
import com.google.gerrit.common.data.PatchSetDetail;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.AccountGeneralPreferences.DiffView;
import com.google.gerrit.reviewdb.client.AccountGeneralPreferences;
import com.google.gerrit.reviewdb.client.AccountGeneralPreferences.DiffView;
import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.Patch;
@@ -218,7 +220,7 @@ public class Dispatcher {
}
private static void select(final String token) {
if (matchPrefix("/q/", token)) {
if (matchPrefix(QUERY, token)) {
query(token);
} else if (matchPrefix("/Documentation/", token)) {
@@ -291,19 +293,19 @@ public class Dispatcher {
}
if (matchExact("mine,starred", token)) {
return PageLinks.toChangeQuery("is:starred");
return toChangeQuery("is:starred");
}
if (matchExact("mine,drafts", token)) {
return PageLinks.toChangeQuery("is:draft");
return toChangeQuery("is:draft");
}
if (matchExact("mine,comments", token)) {
return PageLinks.toChangeQuery("has:draft");
return toChangeQuery("has:draft");
}
if (matchPrefix("mine,watched,", token)) {
return PageLinks.toChangeQuery("is:watched status:open", skip(token));
return toChangeQuery("is:watched status:open");
}
return null;
@@ -311,15 +313,15 @@ public class Dispatcher {
private static String legacyAll(final String token) {
if (matchPrefix("all,abandoned,", token)) {
return PageLinks.toChangeQuery("status:abandoned", skip(token));
return toChangeQuery("status:abandoned");
}
if (matchPrefix("all,merged,", token)) {
return PageLinks.toChangeQuery("status:merged", skip(token));
return toChangeQuery("status:merged");
}
if (matchPrefix("all,open,", token)) {
return PageLinks.toChangeQuery("status:open", skip(token));
return toChangeQuery("status:open");
}
return null;
@@ -330,27 +332,21 @@ public class Dispatcher {
final String s = skip(token);
final int c = s.indexOf(',');
Project.NameKey proj = Project.NameKey.parse(s.substring(0, c));
return PageLinks.toChangeQuery( //
"status:open " + op("project", proj.get()), //
s.substring(c + 1));
return toChangeQuery("status:open " + op("project", proj.get()));
}
if (matchPrefix("project,merged,", token)) {
final String s = skip(token);
final int c = s.indexOf(',');
Project.NameKey proj = Project.NameKey.parse(s.substring(0, c));
return PageLinks.toChangeQuery( //
"status:merged " + op("project", proj.get()), //
s.substring(c + 1));
return toChangeQuery("status:merged " + op("project", proj.get()));
}
if (matchPrefix("project,abandoned,", token)) {
final String s = skip(token);
final int c = s.indexOf(',');
Project.NameKey proj = Project.NameKey.parse(s.substring(0, c));
return PageLinks.toChangeQuery( //
"status:abandoned " + op("project", proj.get()), //
s.substring(c + 1));
return toChangeQuery("status:abandoned " + op("project", proj.get()));
}
return null;
@@ -408,10 +404,22 @@ public class Dispatcher {
return null;
}
private static void query(final String token) {
final String s = skip(token);
final int c = s.indexOf(',');
Gerrit.display(token, new QueryScreen(s.substring(0, c), s.substring(c + 1)));
private static void query(String token) {
String s = skip(token);
int c = s.indexOf(',');
Screen screen;
if (c >= 0) {
String prefix = s.substring(0, c);
if (s.substring(c).equals(",n,z")) {
// Respect legacy token with max sortkey.
screen = new QueryScreen(prefix, 0);
} else {
screen = new QueryScreen(prefix, Integer.parseInt(s.substring(c + 1)));
}
} else {
screen = new QueryScreen(s, 0);
}
Gerrit.display(token, screen);
}
private static Screen mine(final String token) {
@@ -462,7 +470,7 @@ public class Dispatcher {
@Override
public void onFailure(Throwable caught) {
if ("default".equals(dashboardId) && RestApi.isNotFound(caught)) {
Gerrit.display(PageLinks.toChangeQuery(
Gerrit.display(toChangeQuery(
PageLinks.projectQuery(new Project.NameKey(project))));
} else {
super.onFailure(caught);

20
gerrit-gwtui/src/main/java/com/google/gerrit/client/changes/ChangeList.java
View File

@@ -51,30 +51,16 @@ public class ChangeList extends JsArray<ChangeInfo> {
call.get(callback);
}
public static void prev(String query,
int limit, String sortkey,
AsyncCallback<ChangeList> callback) {
RestApi call = newQuery(query);
if (limit > 0) {
call.addParameter("n", limit);
}
addOptions(call, EnumSet.of(ListChangesOption.LABELS));
if (!PagedSingleListScreen.MIN_SORTKEY.equals(sortkey)) {
call.addParameter("P", sortkey);
}
call.get(callback);
}
public static void next(String query,
int limit, String sortkey,
int start, int limit,
AsyncCallback<ChangeList> callback) {
RestApi call = newQuery(query);
if (limit > 0) {
call.addParameter("n", limit);
}
addOptions(call, EnumSet.of(ListChangesOption.LABELS));
if (!PagedSingleListScreen.MAX_SORTKEY.equals(sortkey)) {
call.addParameter("N", sortkey);
if (start != 0) {
call.addParameter("S", start);
}
call.get(callback);
}

2
gerrit-gwtui/src/main/java/com/google/gerrit/client/changes/DashboardTable.java
View File

@@ -95,7 +95,7 @@ public class DashboardTable extends ChangeTable2 {
if (queries.size() == 1) {
ChangeList.next(queries.get(0),
0, PagedSingleListScreen.MAX_SORTKEY,
0, 0,
new GerritCallback<ChangeList>() {
@Override
public void onSuccess(ChangeList result) {

58
gerrit-gwtui/src/main/java/com/google/gerrit/client/changes/PagedSingleListScreen.java
View File

@@ -26,25 +26,19 @@ import com.google.gwt.user.client.ui.HorizontalPanel;
import com.google.gwtexpui.globalkey.client.KeyCommand;
public abstract class PagedSingleListScreen extends Screen {
protected static final String MIN_SORTKEY = "";
protected static final String MAX_SORTKEY = "z";
protected final int pageSize;
protected final int start;
private final String anchorPrefix;
protected ChangeList changes;
private ChangeTable2 table;
private ChangeTable2.Section section;
protected Hyperlink prev;
protected Hyperlink next;
protected ChangeList changes;
private Hyperlink prev;
private Hyperlink next;
protected final String anchorPrefix;
protected boolean useLoadPrev;
protected String pos;
protected PagedSingleListScreen(final String anchorToken,
final String positionToken) {
protected PagedSingleListScreen(String anchorToken, int start) {
anchorPrefix = anchorToken;
useLoadPrev = positionToken.startsWith("p,");
pos = positionToken.substring(2);
this.start = start;
if (Gerrit.isSignedIn()) {
final AccountGeneralPreferences p =
@@ -95,26 +89,12 @@ public abstract class PagedSingleListScreen extends Screen {
add(buttons);
}
@Override
protected void onLoad() {
super.onLoad();
if (useLoadPrev) {
loadPrev();
} else {
loadNext();
}
}
@Override
public void registerKeys() {
super.registerKeys();
table.setRegisterKeys(true);
}
protected abstract void loadPrev();
protected abstract void loadNext();
protected AsyncCallback<ChangeList> loadCallback() {
return new ScreenLoadCallback<ChangeList>(this) {
@Override
@@ -124,22 +104,20 @@ public abstract class PagedSingleListScreen extends Screen {
};
}
protected void display(final ChangeList result) {
protected void display(ChangeList result) {
changes = result;
if (changes.length() != 0) {
final ChangeInfo f = changes.get(0);
final ChangeInfo l = changes.get(changes.length() - 1);
prev.setTargetHistoryToken(anchorPrefix + ",p," + f._sortkey());
next.setTargetHistoryToken(anchorPrefix + ",n," + l._sortkey());
if (useLoadPrev) {
prev.setVisible(f._more_changes());
next.setVisible(!MIN_SORTKEY.equals(pos));
if (start > 0) {
int p = start - pageSize;
prev.setTargetHistoryToken(anchorPrefix + (p > 0 ? "," + p : ""));
prev.setVisible(true);
} else {
prev.setVisible(!MAX_SORTKEY.equals(pos));
next.setVisible(l._more_changes());
prev.setVisible(false);
}
int n = start + changes.length();
next.setTargetHistoryToken(anchorPrefix + "," + n);
next.setVisible(changes.get(changes.length() - 1)._more_changes());
}
table.updateColumnsForLabels(result);
section.display(result);

20
gerrit-gwtui/src/main/java/com/google/gerrit/client/changes/QueryScreen.java
View File

@@ -25,17 +25,17 @@ import com.google.gwtorm.client.KeyUtil;
public class QueryScreen extends PagedSingleListScreen implements
ChangeListScreen {
public static QueryScreen forQuery(String query) {
return forQuery(query, PageLinks.TOP);
return forQuery(query, 0);
}
public static QueryScreen forQuery(String query, String position) {
return new QueryScreen(KeyUtil.encode(query), position);
public static QueryScreen forQuery(String query, int start) {
return new QueryScreen(KeyUtil.encode(query), start);
}
private final String query;
public QueryScreen(final String encQuery, final String positionToken) {
super("/q/" + encQuery, positionToken);
public QueryScreen(String encQuery, int start) {
super(PageLinks.QUERY + encQuery, start);
query = KeyUtil.decode(encQuery);
}
@@ -72,13 +72,9 @@ public class QueryScreen extends PagedSingleListScreen implements
}
@Override
protected void loadPrev() {
ChangeList.prev(query, pageSize, pos, loadCallback());
}
@Override
protected void loadNext() {
ChangeList.next(query, pageSize, pos, loadCallback());
protected void onLoad() {
super.onLoad();
ChangeList.next(query, start, pageSize, loadCallback());
}
private static boolean isSingleQuery(String query) {

4
gerrit-httpd/src/main/java/com/google/gerrit/httpd/DirectChangeByCommit.java
View File

@@ -58,9 +58,9 @@ class DirectChangeByCommit extends HttpServlet {
q = Predicate.and(q, builder.sortkey_before("z"), builder.limit(2), visibleToMe);
ChangeQueryRewriter rewriter = queryRewriter.get();
Predicate<ChangeData> s = rewriter.rewrite(q);
Predicate<ChangeData> s = rewriter.rewrite(q, 0);
if (!(s instanceof ChangeDataSource)) {
s = rewriter.rewrite(Predicate.and(builder.status_open(), q));
s = rewriter.rewrite(Predicate.and(builder.status_open(), q), 0);
}
if (s instanceof ChangeDataSource) {

6
gerrit-httpd/src/main/java/com/google/gerrit/httpd/gitweb/GitWebServlet.java
View File

@@ -261,15 +261,15 @@ class GitWebServlet extends HttpServlet {
p.print(" my $h = shift;\n");
p.print(" my $q;\n");
p.print(" if (!$h || $h eq 'HEAD') {\n");
p.print(" $q = qq{#q,project:$ENV{'GERRIT_PROJECT_NAME'},n,z};\n");
p.print(" $q = qq{#q,project:$ENV{'GERRIT_PROJECT_NAME'}};\n");
p.print(" } elsif ($h =~ /^refs\\/heads\\/([-\\w]+)$/) {\n");
p.print(" $q = qq{#q,project:$ENV{'GERRIT_PROJECT_NAME'}");
p.print("+branch:$1,n,z};\n"); // wrapped
p.print("+branch:$1};\n"); // wrapped
p.print(" } elsif ($h =~ /^refs\\/changes\\/\\d{2}\\/(\\d+)\\/\\d+$/) ");
p.print("{\n"); // wrapped
p.print(" $q = qq{#/c/$1};\n");
p.print(" } else {\n");
p.print(" $q = qq{#/q/$h,n,z};\n");
p.print(" $q = qq{#/q/$h};\n");
p.print(" }\n");
p.print(" my $r = qq{$ENV{'GERRIT_CONTEXT_PATH'}$q};\n");
p.print(" push @{$feature{'actions'}{'default'}},\n");

64
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@@ -48,6 +48,7 @@ import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gerrit.server.query.change.ChangeQueryBuilder;
import com.google.gerrit.server.query.change.SortKeyPredicate;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Provider;
@@ -300,8 +301,8 @@ public class LuceneChangeIndex implements ChangeIndex {
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p, int limit)
throws QueryParseException {
public ChangeDataSource getSource(Predicate<ChangeData> p, int start,
int limit) throws QueryParseException {
Set<Change.Status> statuses = IndexRewriteImpl.getPossibleStatus(p);
List<SubIndex> indexes = Lists.newArrayListWithCapacity(2);
if (!Sets.intersection(statuses, OPEN_STATUSES).isEmpty()) {
@@ -310,8 +311,8 @@ public class LuceneChangeIndex implements ChangeIndex {
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(closedIndex);
}
return new QuerySource(indexes, queryBuilder.toQuery(p), limit,
ChangeQueryBuilder.hasNonTrivialSortKeyAfter(schema, p));
return new QuerySource(indexes, queryBuilder.toQuery(p), start, limit,
getSort(schema, p));
}
@Override
@@ -322,18 +323,38 @@ public class LuceneChangeIndex implements ChangeIndex {
private static final ImmutableSet<String> FIELDS =
ImmutableSet.of(ID_FIELD, CHANGE_FIELD, APPROVAL_FIELD);
@SuppressWarnings("deprecation")
private static Sort getSort(Schema<ChangeData> schema,
Predicate<ChangeData> p) {
// Standard order is descending by sort key, unless reversed due to a
// sortkey_before predicate.
if (SortKeyPredicate.hasSortKeyField(schema)) {
boolean reverse = ChangeQueryBuilder.hasNonTrivialSortKeyAfter(schema, p);
return new Sort(new SortField(
ChangeField.SORTKEY.getName(), SortField.Type.LONG, !reverse));
} else {
return new Sort(
new SortField(
ChangeField.UPDATED.getName(), SortField.Type.LONG, true),
new SortField(
ChangeField.LEGACY_ID.getName(), SortField.Type.INT, true));
}
}
private class QuerySource implements ChangeDataSource {
private final List<SubIndex> indexes;
private final Query query;
private final int start;
private final int limit;
private final boolean reverse;
private final Sort sort;
private QuerySource(List<SubIndex> indexes, Query query, int limit,
boolean reverse) {
private QuerySource(List<SubIndex> indexes, Query query, int start,
int limit, Sort sort) {
this.indexes = indexes;
this.query = query;
this.start = start;
this.limit = limit;
this.reverse = reverse;
this.sort = sort;
}
@Override
@@ -354,24 +375,19 @@ public class LuceneChangeIndex implements ChangeIndex {
@Override
public ResultSet<ChangeData> read() throws OrmException {
IndexSearcher[] searchers = new IndexSearcher[indexes.size()];
Sort sort = new Sort(
new SortField(
ChangeField.SORTKEY.getName(),
SortField.Type.LONG,
// Standard order is descending by sort key, unless reversed due
// to a sortkey_before predicate.
!reverse));
try {
int realLimit = start + limit;
TopDocs[] hits = new TopDocs[indexes.size()];
for (int i = 0; i < indexes.size(); i++) {
searchers[i] = indexes.get(i).acquire();
hits[i] = searchers[i].search(query, limit, sort);
hits[i] = searchers[i].search(query, realLimit, sort);
}
TopDocs docs = TopDocs.merge(sort, limit, hits);
TopDocs docs = TopDocs.merge(sort, realLimit, hits);
List<ChangeData> result =
Lists.newArrayListWithCapacity(docs.scoreDocs.length);
for (ScoreDoc sd : docs.scoreDocs) {
for (int i = start; i < docs.scoreDocs.length; i++) {
ScoreDoc sd = docs.scoreDocs[i];
Document doc = searchers[sd.shardIndex].doc(sd.doc, FIELDS);
result.add(toChangeData(doc));
}
@@ -462,9 +478,17 @@ public class LuceneChangeIndex implements ChangeIndex {
doc.add(new LongField(name, (Long) value, store));
}
} else if (type == FieldType.TIMESTAMP) {
@SuppressWarnings("deprecation")
boolean legacy = values.getField() == ChangeField.LEGACY_UPDATED;
if (legacy) {
for (Object value : values.getValues()) {
int t = QueryBuilder.toIndexTime((Timestamp) value);
doc.add(new IntField(name, t, store));
int t = queryBuilder.toIndexTimeInMinutes((Timestamp) value);
doc.add(new IntField(name, (int) t, store));
}
} else {
for (Object value : values.getValues()) {
doc.add(new LongField(name, ((Timestamp) value).getTime(), store));
}
}
} else if (type == FieldType.EXACT
|| type == FieldType.PREFIX) {

26
gerrit-lucene/src/main/java/com/google/gerrit/lucene/QueryBuilder.java
View File

@@ -179,28 +179,46 @@ public class QueryBuilder {
false, false);
}
@SuppressWarnings("deprecation")
private Query timestampQuery(IndexPredicate<ChangeData> p)
throws QueryParseException {
if (p instanceof TimestampRangePredicate) {
TimestampRangePredicate<ChangeData> r =
(TimestampRangePredicate<ChangeData>) p;
if (r.getField() == ChangeField.LEGACY_UPDATED) {
return NumericRangeQuery.newIntRange(
r.getField().getName(),
toIndexTime(r.getMinTimestamp()),
toIndexTime(r.getMaxTimestamp()),
toIndexTimeInMinutes(r.getMinTimestamp()),
toIndexTimeInMinutes(r.getMaxTimestamp()),
true, true);
} else {
return NumericRangeQuery.newLongRange(
r.getField().getName(),
r.getMinTimestamp().getTime(),
r.getMaxTimestamp().getTime(),
true, true);
}
}
throw new QueryParseException("not a timestamp: " + p);
}
@SuppressWarnings("deprecation")
private Query notTimestamp(TimestampRangePredicate<ChangeData> r)
throws QueryParseException {
if (r.getMinTimestamp().getTime() == 0) {
if (r.getField() == ChangeField.LEGACY_UPDATED) {
return NumericRangeQuery.newIntRange(
r.getField().getName(),
toIndexTime(r.getMaxTimestamp()),
toIndexTimeInMinutes(r.getMaxTimestamp()),
null,
true, true);
} else {
return NumericRangeQuery.newLongRange(
r.getField().getName(),
r.getMaxTimestamp().getTime(),
null,
true, true);
}
}
throw new QueryParseException("cannot negate: " + r);
}
@@ -232,7 +250,7 @@ public class QueryBuilder {
return queryBuilder.createPhraseQuery(p.getField().getName(), p.getValue());
}
public static int toIndexTime(Date ts) {
public int toIndexTimeInMinutes(Date ts) {
return (int) (ts.getTime() / 60000);
}

2
gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java
View File

@@ -86,7 +86,7 @@ public class ChangeUtil {
* We overrun approximately 4,083 years later, so ~6092.
*/
@VisibleForTesting
public static final long SORT_KEY_EPOCH_MINS =
private static final long SORT_KEY_EPOCH_MINS =
MINUTES.convert(1222819200L, SECONDS);
private static final Object uuidLock = new Object();

17
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java
View File

@@ -115,10 +115,24 @@ public class ChangeField {
}
};
// Same value as UPDATED, but implementations truncated to minutes.
@Deprecated
/** Last update time since January 1, 1970. */
public static final FieldDef<ChangeData, Timestamp> LEGACY_UPDATED =
new FieldDef.Single<ChangeData, Timestamp>(
"updated", FieldType.TIMESTAMP, true) {
@Override
public Timestamp get(ChangeData input, FillArgs args)
throws OrmException {
return input.change().getLastUpdatedOn();
}
};
/** Last update time since January 1, 1970. */
public static final FieldDef<ChangeData, Timestamp> UPDATED =
new FieldDef.Single<ChangeData, Timestamp>(
"updated", FieldType.TIMESTAMP, true) {
"updated2", FieldType.TIMESTAMP, true) {
@Override
public Timestamp get(ChangeData input, FillArgs args)
throws OrmException {
@@ -152,6 +166,7 @@ public class ChangeField {
* Redundant with {@link #UPDATED} and {@link #LEGACY_ID}, but secondary index
* implementations may not be able to search over tuples of values.
*/
@Deprecated
public static final FieldDef<ChangeData, Long> SORTKEY =
new FieldDef.Single<ChangeData, Long>(
"sortkey2", FieldType.LONG, true) {

5
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeIndex.java
View File

@@ -91,6 +91,7 @@ public interface ChangeIndex {
* @param p the predicate to match. Must be a tree containing only AND, OR,
* or NOT predicates as internal nodes, and {@link IndexPredicate}s as
* leaves.
* @param start offset in results list at which to start returning results.
* @param limit maximum number of results to return.
* @return a source of documents matching the predicate. Documents must be
* returned in descending sort key order, unless a {@code sortkey_after}
@@ -101,8 +102,8 @@ public interface ChangeIndex {
* @throws QueryParseException if the predicate could not be converted to an
* indexed data source.
*/
public ChangeDataSource getSource(Predicate<ChangeData> p, int limit)
throws QueryParseException;
public ChangeDataSource getSource(Predicate<ChangeData> p, int start,
int limit) throws QueryParseException;
/**
* Mark whether this index is up-to-date and ready to serve reads.

35
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java
View File

@@ -38,7 +38,7 @@ public class ChangeSchemas {
ChangeField.PROJECT,
ChangeField.REF,
ChangeField.TOPIC,
ChangeField.UPDATED,
ChangeField.LEGACY_UPDATED,
ChangeField.LEGACY_SORTKEY,
ChangeField.PATH,
ChangeField.OWNER,
@@ -58,7 +58,7 @@ public class ChangeSchemas {
ChangeField.PROJECT,
ChangeField.REF,
ChangeField.TOPIC,
ChangeField.UPDATED,
ChangeField.LEGACY_UPDATED,
ChangeField.LEGACY_SORTKEY,
ChangeField.PATH,
ChangeField.OWNER,
@@ -72,6 +72,7 @@ public class ChangeSchemas {
ChangeField.CHANGE,
ChangeField.APPROVAL);
@SuppressWarnings("deprecation")
static final Schema<ChangeData> V3 = release(
ChangeField.LEGACY_ID,
ChangeField.ID,
@@ -79,7 +80,7 @@ public class ChangeSchemas {
ChangeField.PROJECT,
ChangeField.REF,
ChangeField.TOPIC,
ChangeField.UPDATED,
ChangeField.LEGACY_UPDATED,
ChangeField.SORTKEY,
ChangeField.PATH,
ChangeField.OWNER,
@@ -96,6 +97,7 @@ public class ChangeSchemas {
// For upgrade to Lucene 4.4.0 index format only.
static final Schema<ChangeData> V4 = release(V3.getFields().values());
@SuppressWarnings("deprecation")
static final Schema<ChangeData> V5 = release(
ChangeField.LEGACY_ID,
ChangeField.ID,
@@ -103,7 +105,7 @@ public class ChangeSchemas {
ChangeField.PROJECT,
ChangeField.REF,
ChangeField.TOPIC,
ChangeField.UPDATED,
ChangeField.LEGACY_UPDATED,
ChangeField.SORTKEY,
ChangeField.PATH,
ChangeField.OWNER,
@@ -121,6 +123,7 @@ public class ChangeSchemas {
// For upgrade to Lucene 4.6.0 index format only.
static final Schema<ChangeData> V6 = release(V5.getFields().values());
@SuppressWarnings("deprecation")
static final Schema<ChangeData> V7 = release(
ChangeField.LEGACY_ID,
ChangeField.ID,
@@ -128,7 +131,7 @@ public class ChangeSchemas {
ChangeField.PROJECT,
ChangeField.REF,
ChangeField.TOPIC,
ChangeField.UPDATED,
ChangeField.LEGACY_UPDATED,
ChangeField.SORTKEY,
ChangeField.FILE_PART,
ChangeField.PATH,
@@ -144,6 +147,28 @@ public class ChangeSchemas {
ChangeField.APPROVAL,
ChangeField.MERGEABLE);
static final Schema<ChangeData> V8 = release(
ChangeField.LEGACY_ID,
ChangeField.ID,
ChangeField.STATUS,
ChangeField.PROJECT,
ChangeField.REF,
ChangeField.TOPIC,
ChangeField.UPDATED,
ChangeField.FILE_PART,
ChangeField.PATH,
ChangeField.OWNER,
ChangeField.REVIEWER,
ChangeField.COMMIT,
ChangeField.TR,
ChangeField.LABEL,
ChangeField.REVIEWED,
ChangeField.COMMIT_MESSAGE,
ChangeField.COMMENT,
ChangeField.CHANGE,
ChangeField.APPROVAL,
ChangeField.MERGEABLE);
private static Schema<ChangeData> release(Collection<FieldDef<ChangeData, ?>> fields) {
return new Schema<ChangeData>(true, fields);

13
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexRewriteImpl.java
View File

@@ -15,6 +15,7 @@
package com.google.gerrit.server.index;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.gerrit.reviewdb.client.Change;
@@ -130,13 +131,17 @@ public class IndexRewriteImpl implements ChangeQueryRewriter {
}
@Override
public Predicate<ChangeData> rewrite(Predicate<ChangeData> in)
public Predicate<ChangeData> rewrite(Predicate<ChangeData> in, int start)
throws QueryParseException {
ChangeIndex index = indexes.getSearchIndex();
in = basicRewrites.rewrite(in);
int limit = Math.max(1, ChangeQueryBuilder.hasLimit(in)
? ChangeQueryBuilder.getLimit(in)
: MAX_LIMIT);
int limit =
Objects.firstNonNull(ChangeQueryBuilder.getLimit(in), MAX_LIMIT);
// Increase the limit rather than skipping, since we don't know how many
// skipped results would have been filtered out by the enclosing AndSource.
limit += start;
limit = Math.max(limit, 1);
limit = Math.min(limit, MAX_LIMIT);
Predicate<ChangeData> out = rewriteImpl(in, index, limit);
if (in == out || out instanceof IndexPredicate) {

17
gerrit-server/src/main/java/com/google/gerrit/server/index/IndexedChangeQuery.java
View File

@@ -89,7 +89,7 @@ public class IndexedChangeQuery extends Predicate<ChangeData>
this.index = index;
this.limit = limit;
this.pred = pred;
this.source = index.getSource(pred, limit);
this.source = index.getSource(pred, 0, limit);
}
@Override
@@ -166,7 +166,20 @@ public class IndexedChangeQuery extends Predicate<ChangeData>
public ResultSet<ChangeData> restart(ChangeData last) throws OrmException {
pred = replaceSortKeyPredicates(pred, last.change().getSortKey());
try {
source = index.getSource(pred, limit);
source = index.getSource(pred, 0, limit);
} catch (QueryParseException e) {
// Don't need to show this exception to the user; the only thing that
// changed about pred was its SortKeyPredicates, and any other QPEs
// that might happen should have already thrown from the constructor.
throw new OrmException(e);
}
return read();
}
@Override
public ResultSet<ChangeData> restart(int start) throws OrmException {
try {
source = index.getSource(pred, start, limit);
} catch (QueryParseException e) {
// Don't need to show this exception to the user; the only thing that
// changed about pred was its SortKeyPredicates, and any other QPEs

21
gerrit-server/src/main/java/com/google/gerrit/server/index/TimestampRangePredicate.java
View File

@@ -14,7 +14,12 @@
package com.google.gerrit.server.index;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.gerrit.server.index.ChangeField.UPDATED;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import org.eclipse.jgit.util.GitDateParser;
import org.joda.time.DateTime;
@@ -25,6 +30,22 @@ import java.util.Date;
import java.util.Locale;
public abstract class TimestampRangePredicate<I> extends IndexPredicate<I> {
@SuppressWarnings({"deprecation", "unchecked"})
protected static FieldDef<ChangeData, Timestamp> updatedField(
Schema<ChangeData> schema) {
if (schema == null) {
return ChangeField.LEGACY_UPDATED;
}
FieldDef<ChangeData, ?> f = schema.getFields().get(UPDATED.getName());
if (f == null) {
f = schema.getFields().get(ChangeField.LEGACY_UPDATED.getName());
checkNotNull(f, "schema missing updated field, found: %s", schema);
}
checkArgument(f.getType() == FieldType.TIMESTAMP,
"expected %s to be TIMESTAMP, found %s", f.getName(), f.getType());
return (FieldDef<ChangeData, Timestamp>) f;
}
protected static Date parse(String value) throws QueryParseException {
try {
return GitDateParser.parse(value, DateTime.now().toCalendar(Locale.US));

7
gerrit-server/src/main/java/com/google/gerrit/server/query/change/AfterPredicate.java
View File

@@ -14,7 +14,7 @@
package com.google.gerrit.server.query.change;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.TimestampRangePredicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gwtorm.server.OrmException;
@@ -24,8 +24,9 @@ import java.util.Date;
public class AfterPredicate extends TimestampRangePredicate<ChangeData> {
private final Date cut;
AfterPredicate(String value) throws QueryParseException {
super(ChangeField.UPDATED, ChangeQueryBuilder.FIELD_AFTER, value);
AfterPredicate(Schema<ChangeData> schema, String value)
throws QueryParseException {
super(updatedField(schema), ChangeQueryBuilder.FIELD_BEFORE, value);
cut = parse(value);
}

10
gerrit-server/src/main/java/com/google/gerrit/server/query/change/AgePredicate.java
View File

@@ -19,7 +19,7 @@ import static java.util.concurrent.TimeUnit.SECONDS;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.config.ConfigUtil;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.TimestampRangePredicate;
import com.google.gerrit.server.util.TimeUtil;
import com.google.gwtorm.server.OrmException;
@@ -29,8 +29,8 @@ import java.sql.Timestamp;
public class AgePredicate extends TimestampRangePredicate<ChangeData> {
private final long cut;
AgePredicate(String value) {
super(ChangeField.UPDATED, ChangeQueryBuilder.FIELD_AGE, value);
AgePredicate(Schema<ChangeData> schema, String value) {
super(updatedField(schema), ChangeQueryBuilder.FIELD_AGE, value);
long s = ConfigUtil.getTimeUnit(getValue(), 0, SECONDS);
long ms = MILLISECONDS.convert(s, SECONDS);
@@ -47,10 +47,6 @@ public class AgePredicate extends TimestampRangePredicate<ChangeData> {
return new Timestamp(cut);
}
long getCut() {
return cut + 1;
}
@Override
public boolean match(final ChangeData object) throws OrmException {
Change change = object.change();

30
gerrit-server/src/main/java/com/google/gerrit/server/query/change/AndSource.java
View File

@@ -14,9 +14,12 @@
package com.google.gerrit.server.query.change;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.gerrit.server.query.AndPredicate;
@@ -71,10 +74,18 @@ public class AndSource extends AndPredicate<ChangeData>
return r;
}
private final int start;
private int cardinality = -1;
public AndSource(Collection<? extends Predicate<ChangeData>> that) {
this(that, 0);
}
public AndSource(Collection<? extends Predicate<ChangeData>> that,
int start) {
super(sort(that));
checkArgument(start >= 0, "negative start: %s", start);
this.start = start;
}
@Override
@@ -98,9 +109,13 @@ public class AndSource extends AndPredicate<ChangeData>
if (source == null) {
throw new OrmException("No ChangeDataSource: " + this);
}
@SuppressWarnings("unchecked")
Predicate<ChangeData> pred = (Predicate<ChangeData>) source;
boolean useSortKey = ChangeQueryBuilder.hasSortKey(pred);
List<ChangeData> r = Lists.newArrayList();
ChangeData last = null;
int nextStart = 0;
boolean skipped = false;
for (ChangeData data : buffer(source, source.read())) {
if (match(data)) {
@@ -109,6 +124,7 @@ public class AndSource extends AndPredicate<ChangeData>
skipped = true;
}
last = data;
nextStart++;
}
if (skipped && last != null && source instanceof Paginated) {
@@ -117,21 +133,31 @@ public class AndSource extends AndPredicate<ChangeData>
// limit the caller wants. Restart the source and continue.
//
Paginated p = (Paginated) source;
while (skipped && r.size() < p.limit()) {
while (skipped && r.size() < p.limit() + start) {
ChangeData lastBeforeRestart = last;
skipped = false;
last = null;
for (ChangeData data : buffer(source, p.restart(lastBeforeRestart))) {
ResultSet<ChangeData> next = useSortKey
? p.restart(lastBeforeRestart)
: p.restart(nextStart);
for (ChangeData data : buffer(source, next)) {
if (match(data)) {
r.add(data);
} else {
skipped = true;
}
last = data;
nextStart++;
}
}
}
if (start >= r.size()) {
r = ImmutableList.of();
} else if (start > 0) {
r = ImmutableList.copyOf(r.subList(start, r.size()));
}
return new ListResultSet<ChangeData>(r);
}

37
gerrit-server/src/main/java/com/google/gerrit/server/query/change/BasicChangeRewrites.java
View File

@@ -14,13 +14,8 @@
package com.google.gerrit.server.query.change;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.ChangeUtil;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.IndexCollection;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.query.IntPredicate;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryRewriter;
@@ -41,19 +36,12 @@ public class BasicChangeRewrites extends QueryRewriter<ChangeData> {
new QueryRewriter.Definition<ChangeData, BasicChangeRewrites>(
BasicChangeRewrites.class, BUILDER);
static Schema<ChangeData> schema(@Nullable IndexCollection indexes) {
ChangeIndex index = indexes != null ? indexes.getSearchIndex() : null;
return index != null ? index.getSchema() : null;
}
protected final Provider<ReviewDb> dbProvider;
private final IndexCollection indexes;
@Inject
public BasicChangeRewrites(Provider<ReviewDb> dbProvider, IndexCollection indexes) {
public BasicChangeRewrites(Provider<ReviewDb> dbProvider) {
super(mydef);
this.dbProvider = dbProvider;
this.indexes = indexes;
}
@Rewrite("-status:open")
@@ -84,14 +72,6 @@ public class BasicChangeRewrites extends QueryRewriter<ChangeData> {
new ChangeStatusPredicate(Change.Status.MERGED));
}
@SuppressWarnings("unchecked")
@NoCostComputation
@Rewrite("sortkey_before:z A=(age:*)")
public Predicate<ChangeData> r00_ageToSortKey(@Named("A") AgePredicate a) {
String cut = ChangeUtil.sortKey(a.getCut(), Integer.MAX_VALUE);
return and(new SortKeyPredicate.Before(schema(indexes), dbProvider, cut), a);
}
@NoCostComputation
@Rewrite("A=(limit:*) B=(limit:*)")
public Predicate<ChangeData> r00_smallestLimit(
@@ -100,21 +80,6 @@ public class BasicChangeRewrites extends QueryRewriter<ChangeData> {
return a.intValue() <= b.intValue() ? a : b;
}
@NoCostComputation
@Rewrite("A=(sortkey_before:*) B=(sortkey_before:*)")
public Predicate<ChangeData> r00_oldestSortKey(
@Named("A") SortKeyPredicate.Before a,
@Named("B") SortKeyPredicate.Before b) {
return a.getValue().compareTo(b.getValue()) <= 0 ? a : b;
}
@NoCostComputation
@Rewrite("A=(sortkey_after:*) B=(sortkey_after:*)")
public Predicate<ChangeData> r00_newestSortKey(
@Named("A") SortKeyPredicate.After a, @Named("B") SortKeyPredicate.After b) {
return a.getValue().compareTo(b.getValue()) >= 0 ? a : b;
}
private static final class InvalidProvider<T> implements Provider<T> {
@Override
public T get() {

7
gerrit-server/src/main/java/com/google/gerrit/server/query/change/BeforePredicate.java
View File

@@ -14,7 +14,7 @@
package com.google.gerrit.server.query.change;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.TimestampRangePredicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gwtorm.server.OrmException;
@@ -24,8 +24,9 @@ import java.util.Date;
public class BeforePredicate extends TimestampRangePredicate<ChangeData> {
private final Date cut;
BeforePredicate(String value) throws QueryParseException {
super(ChangeField.UPDATED, ChangeQueryBuilder.FIELD_BEFORE, value);
BeforePredicate(Schema<ChangeData> schema, String value)
throws QueryParseException {
super(updatedField(schema), ChangeQueryBuilder.FIELD_BEFORE, value);
cut = parse(value);
}

33
gerrit-server/src/main/java/com/google/gerrit/server/query/change/ChangeQueryBuilder.java
View File

@@ -16,6 +16,7 @@ package com.google.gerrit.server.query.change;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.common.data.GroupReference;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.AccountGroup;
@@ -124,13 +125,10 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
ChangeQueryBuilder.class);
@SuppressWarnings("unchecked")
public static boolean hasLimit(Predicate<ChangeData> p) {
return find(p, IntPredicate.class, FIELD_LIMIT) != null;
}
@SuppressWarnings("unchecked")
public static int getLimit(Predicate<ChangeData> p) {
return ((IntPredicate<?>) find(p, IntPredicate.class, FIELD_LIMIT)).intValue();
public static Integer getLimit(Predicate<ChangeData> p) {
IntPredicate<?> ip =
(IntPredicate<?>) find(p, IntPredicate.class, FIELD_LIMIT);
return ip != null ? ip.intValue() : null;
}
public static boolean hasNonTrivialSortKeyAfter(Schema<ChangeData> schema,
@@ -237,12 +235,12 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
@Operator
public Predicate<ChangeData> age(String value) {
return new AgePredicate(value);
return new AgePredicate(schema(args.indexes), value);
}
@Operator
public Predicate<ChangeData> before(String value) throws QueryParseException {
return new BeforePredicate(value);
return new BeforePredicate(schema(args.indexes), value);
}
@Operator
@@ -252,7 +250,7 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
@Operator
public Predicate<ChangeData> after(String value) throws QueryParseException {
return new AfterPredicate(value);
return new AfterPredicate(schema(args.indexes), value);
}
@Operator
@@ -660,16 +658,18 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
return new LimitPredicate(limit);
}
boolean supportsSortKey() {
return SortKeyPredicate.hasSortKeyField(schema(args.indexes));
}
@Operator
public Predicate<ChangeData> sortkey_after(String sortKey) {
return new SortKeyPredicate.After(
BasicChangeRewrites.schema(args.indexes), args.db, sortKey);
return new SortKeyPredicate.After(schema(args.indexes), args.db, sortKey);
}
@Operator
public Predicate<ChangeData> sortkey_before(String sortKey) {
return new SortKeyPredicate.Before(
BasicChangeRewrites.schema(args.indexes), args.db, sortKey);
return new SortKeyPredicate.Before(schema(args.indexes), args.db, sortKey);
}
@Operator
@@ -778,4 +778,9 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData> {
}
throw new IllegalArgumentException();
}
private static Schema<ChangeData> schema(@Nullable IndexCollection indexes) {
ChangeIndex index = indexes != null ? indexes.getSearchIndex() : null;
return index != null ? index.getSchema() : null;
}
}

2
gerrit-server/src/main/java/com/google/gerrit/server/query/change/ChangeQueryRewriter.java
View File

@@ -18,6 +18,6 @@ import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
public interface ChangeQueryRewriter {
Predicate<ChangeData> rewrite(Predicate<ChangeData> in)
Predicate<ChangeData> rewrite(Predicate<ChangeData> in, int start)
throws QueryParseException;
}

2
gerrit-server/src/main/java/com/google/gerrit/server/query/change/CommentPredicate.java
View File

@@ -36,7 +36,7 @@ class CommentPredicate extends IndexPredicate<ChangeData> {
public boolean match(ChangeData object) throws OrmException {
try {
for (ChangeData cData : index.getSource(
Predicate.and(new LegacyChangeIdPredicate(args, object.getId()), this), 1)
Predicate.and(new LegacyChangeIdPredicate(args, object.getId()), this), 0, 1)
.read()) {
if (cData.getId().equals(object.getId())) {
return true;

2
gerrit-server/src/main/java/com/google/gerrit/server/query/change/MessagePredicate.java
View File

@@ -40,7 +40,7 @@ class MessagePredicate extends IndexPredicate<ChangeData> {
public boolean match(ChangeData object) throws OrmException {
try {
for (ChangeData cData : index.getSource(
Predicate.and(new LegacyChangeIdPredicate(args, object.getId()), this), 1)
Predicate.and(new LegacyChangeIdPredicate(args, object.getId()), this), 0, 1)
.read()) {
if (cData.getId().equals(object.getId())) {
return true;

2
gerrit-server/src/main/java/com/google/gerrit/server/query/change/Paginated.java
View File

@@ -21,4 +21,6 @@ public interface Paginated {
int limit();
ResultSet<ChangeData> restart(ChangeData last) throws OrmException;
ResultSet<ChangeData> restart(int start) throws OrmException;
}

5
gerrit-server/src/main/java/com/google/gerrit/server/query/change/QueryChanges.java
View File

@@ -80,6 +80,11 @@ public class QueryChanges implements RestReadView<TopLevelResource> {
imp.setSortkeyBefore(key);
}
@Option(name = "-S", metaVar = "CNT", usage = "Number of changes to skip")
public void setStart(int start) {
imp.setStart(start);
}
@Inject
QueryChanges(ChangeJson json, QueryProcessor qp, Provider<CurrentUser> user) {
this.json = json;

24
gerrit-server/src/main/java/com/google/gerrit/server/query/change/QueryProcessor.java
View File

@@ -14,6 +14,7 @@
package com.google.gerrit.server.query.change;
import com.google.common.base.Objects;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.gerrit.common.data.GlobalCapability;
@@ -103,6 +104,7 @@ public class QueryProcessor {
private OutputFormat outputFormat = OutputFormat.TEXT;
private int limit;
private int start;
private String sortkeyAfter;
private String sortkeyBefore;
private boolean includePatchSets;
@@ -144,6 +146,10 @@ public class QueryProcessor {
limit = n;
}
void setStart(int n) {
start = n;
}
void setSortkeyAfter(String sortkey) {
sortkeyAfter = sortkey;
}
@@ -240,17 +246,17 @@ public class QueryProcessor {
List<ChangeDataSource> sources = Lists.newArrayListWithCapacity(cnt);
for (String query : queries) {
Predicate<ChangeData> q = parseQuery(query, visibleToMe);
Predicate<ChangeData> s = queryRewriter.rewrite(q);
Predicate<ChangeData> s = queryRewriter.rewrite(q, start);
if (!(s instanceof ChangeDataSource)) {
q = Predicate.and(queryBuilder.status_open(), q);
s = queryRewriter.rewrite(q);
s = queryRewriter.rewrite(q, start);
}
if (!(s instanceof ChangeDataSource)) {
throw new QueryParseException("invalid query: " + s);
}
// Don't trust QueryRewriter to have left the visible predicate.
AndSource a = new AndSource(ImmutableList.of(s, visibleToMe));
AndSource a = new AndSource(ImmutableList.of(s, visibleToMe), start);
limits.add(limit(q));
sources.add(a);
}
@@ -264,7 +270,11 @@ public class QueryProcessor {
List<List<ChangeData>> out = Lists.newArrayListWithCapacity(cnt);
for (int i = 0; i < cnt; i++) {
List<ChangeData> results = matches.get(i).toList();
Collections.sort(results, sortkeyAfter != null ? cmpAfter : cmpBefore);
if (sortkeyAfter != null) {
Collections.sort(results, cmpAfter);
} else if (sortkeyBefore != null) {
Collections.sort(results, cmpBefore);
}
if (results.size() > maxLimit) {
moreResults = true;
}
@@ -406,16 +416,14 @@ public class QueryProcessor {
}
private int limit(Predicate<ChangeData> s) {
int n = ChangeQueryBuilder.hasLimit(s)
? ChangeQueryBuilder.getLimit(s)
: maxLimit;
int n = Objects.firstNonNull(ChangeQueryBuilder.getLimit(s), maxLimit);
return limit > 0 ? Math.min(n, limit) + 1 : n + 1;
}
private Predicate<ChangeData> parseQuery(String queryString,
final Predicate<ChangeData> visibleToMe) throws QueryParseException {
Predicate<ChangeData> q = queryBuilder.parse(queryString);
if (!ChangeQueryBuilder.hasSortKey(q)) {
if (queryBuilder.supportsSortKey() && !ChangeQueryBuilder.hasSortKey(q)) {
if (sortkeyBefore != null) {
q = Predicate.and(q, queryBuilder.sortkey_before(sortkeyBefore));
} else if (sortkeyAfter != null) {

15
gerrit-server/src/main/java/com/google/gerrit/server/query/change/SortKeyPredicate.java
View File

@@ -29,6 +29,10 @@ import com.google.gwtorm.server.OrmException;
import com.google.inject.Provider;
public abstract class SortKeyPredicate extends IndexPredicate<ChangeData> {
public static boolean hasSortKeyField(Schema<ChangeData> schema) {
return sortkeyFieldOrNull(schema) != null;
}
@SuppressWarnings("deprecation")
private static long parseSortKey(Schema<ChangeData> schema, String value) {
FieldDef<ChangeData, ?> field = schema.getFields().get(SORTKEY.getName());
@@ -40,7 +44,8 @@ public abstract class SortKeyPredicate extends IndexPredicate<ChangeData> {
}
@SuppressWarnings("deprecation")
private static FieldDef<ChangeData, ?> sortkeyField(Schema<ChangeData> schema) {
private static FieldDef<ChangeData, ?> sortkeyFieldOrNull(
Schema<ChangeData> schema) {
if (schema == null) {
return ChangeField.LEGACY_SORTKEY;
}
@@ -48,9 +53,13 @@ public abstract class SortKeyPredicate extends IndexPredicate<ChangeData> {
if (f != null) {
return f;
}
return schema.getFields().get(ChangeField.LEGACY_SORTKEY.getName());
}
private static FieldDef<ChangeData, ?> sortkeyField(Schema<ChangeData> schema) {
return checkNotNull(
schema.getFields().get(ChangeField.LEGACY_SORTKEY.getName()),
"schema missing sortkey field, found: %s", schema.getFields().keySet());
sortkeyFieldOrNull(schema),
"schema missing sortkey field, found: %s", schema);
}
protected final Schema<ChangeData> schema;

6
gerrit-server/src/test/java/com/google/gerrit/server/index/FakeIndex.java
View File

@@ -31,7 +31,7 @@ class FakeIndex implements ChangeIndex {
ImmutableList.of(
ChangeField.STATUS,
ChangeField.PATH,
ChangeField.SORTKEY));
ChangeField.UPDATED));
private static class Source implements ChangeDataSource {
private final Predicate<ChangeData> p;
@@ -88,8 +88,8 @@ class FakeIndex implements ChangeIndex {
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p, int limit)
throws QueryParseException {
public ChangeDataSource getSource(Predicate<ChangeData> p, int start,
int limit) throws QueryParseException {
return new FakeIndex.Source(p);
}

29
gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java
View File

@@ -37,6 +37,7 @@ import com.google.gerrit.server.query.change.OrSource;
import org.junit.Before;
import org.junit.Test;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.Set;
@@ -54,7 +55,7 @@ public class IndexRewriteTest {
queryBuilder = new FakeQueryBuilder(indexes);
rewrite = new IndexRewriteImpl(
indexes,
new BasicChangeRewrites(null, indexes));
new BasicChangeRewrites(null));
}
@Test
@@ -97,7 +98,7 @@ public class IndexRewriteTest {
parse("-status:abandoned (status:open OR status:merged)");
assertEquals(
query(parse("status:new OR status:submitted OR status:draft OR status:merged")),
rewrite.rewrite(in));
rewrite.rewrite(in, 0));
}
@Test
@@ -168,6 +169,23 @@ public class IndexRewriteTest {
out.getChildren());
}
@Test
public void testStartIncreasesLimit() throws Exception {
Predicate<ChangeData> f = parse("file:a");
Predicate<ChangeData> l = parse("limit:3");
Predicate<ChangeData> in = and(f, l);
assertEquals(and(query(f, 3), l), rewrite.rewrite(in, 0));
assertEquals(and(query(f, 4), l), rewrite.rewrite(in, 1));
assertEquals(and(query(f, 5), l), rewrite.rewrite(in, 2));
}
@Test
public void testStartDoesNotExceedMaxLimit() throws Exception {
Predicate<ChangeData> in = parse("file:a");
assertEquals(query(in), rewrite.rewrite(in, 0));
assertEquals(query(in), rewrite.rewrite(in, 1));
}
@Test
public void testGetPossibleStatus() throws Exception {
assertEquals(EnumSet.allOf(Change.Status.class), status("file:a"));
@@ -203,9 +221,14 @@ public class IndexRewriteTest {
return queryBuilder.parse(query);
}
@SafeVarargs
private static AndSource and(Predicate<ChangeData>... preds) {
return new AndSource(Arrays.asList(preds));
}
private Predicate<ChangeData> rewrite(Predicate<ChangeData> in)
throws QueryParseException {
return rewrite.rewrite(in);
return rewrite.rewrite(in, 0);
}
private IndexedChangeQuery query(Predicate<ChangeData> p)

71
gerrit-server/src/test/java/com/google/gerrit/server/index/IndexedChangeQueryTest.java
View File

@@ -1,71 +0,0 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index;
import static com.google.gerrit.server.index.IndexedChangeQuery.replaceSortKeyPredicates;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeQueryBuilder;
import org.junit.Before;
import org.junit.Test;
public class IndexedChangeQueryTest {
private FakeIndex index;
private ChangeQueryBuilder queryBuilder;
@Before
public void setUp() throws Exception {
index = new FakeIndex(FakeIndex.V2);
IndexCollection indexes = new IndexCollection();
indexes.setSearchIndex(index);
queryBuilder = new FakeQueryBuilder(indexes);
}
@Test
public void testReplaceSortKeyPredicate_NoSortKey() throws Exception {
Predicate<ChangeData> p = parse("foo:a bar:b OR (foo:b bar:a)");
assertSame(p, replaceSortKeyPredicates(p, "1234"));
}
@Test
public void testReplaceSortKeyPredicate_TopLevelSortKey() throws Exception {
Predicate<ChangeData> p;
p = parse("foo:a bar:b sortkey_before:1234 OR (foo:b bar:a)");
assertEquals(parse("foo:a bar:b sortkey_before:5678 OR (foo:b bar:a)"),
replaceSortKeyPredicates(p, "5678"));
p = parse("foo:a bar:b sortkey_after:1234 OR (foo:b bar:a)");
assertEquals(parse("foo:a bar:b sortkey_after:5678 OR (foo:b bar:a)"),
replaceSortKeyPredicates(p, "5678"));
}
@Test
public void testReplaceSortKeyPredicate_NestedSortKey() throws Exception {
Predicate<ChangeData> p;
p = parse("foo:a bar:b OR (foo:b bar:a AND sortkey_before:1234)");
assertEquals(parse("foo:a bar:b OR (foo:b bar:a sortkey_before:5678)"),
replaceSortKeyPredicates(p, "5678"));
p = parse("foo:a bar:b OR (foo:b bar:a AND sortkey_after:1234)");
assertEquals(parse("foo:a bar:b OR (foo:b bar:a sortkey_after:5678)"),
replaceSortKeyPredicates(p, "5678"));
}
private Predicate<ChangeData> parse(String query) throws QueryParseException {
return queryBuilder.parse(query);
}
}

18
gerrit-server/src/test/java/com/google/gerrit/server/notedb/ChangeNotesTest.java
View File

@@ -19,9 +19,7 @@ import static com.google.gerrit.server.notedb.ReviewerState.REVIEWER;
import static com.google.gerrit.server.project.Util.category;
import static com.google.gerrit.server.project.Util.value;
import static com.google.inject.Scopes.SINGLETON;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.easymock.EasyMock.expect;
import static org.junit.Assert.assertEquals;
@@ -38,7 +36,6 @@ import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.PatchSetApproval;
import com.google.gerrit.reviewdb.client.PatchSetInfo;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.ChangeUtil;
import com.google.gerrit.server.GerritPersonIdent;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.account.AccountCache;
@@ -73,6 +70,7 @@ import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.joda.time.DateTime;
import org.joda.time.DateTimeUtils;
import org.joda.time.DateTimeUtils.MillisProvider;
import org.junit.After;
@@ -107,11 +105,12 @@ public class ChangeNotesTest {
private IdentifiedUser changeOwner;
private IdentifiedUser otherUser;
private Injector injector;
private String systemTimeZone;
private volatile long clockStepMs;
@Before
public void setUp() throws Exception {
setMillisProvider();
setTimeForTesting();
serverIdent = new PersonIdent(
"Gerrit Server", "noreply@gerrit.com", TimeUtil.nowTs(), TZ);
@@ -159,11 +158,11 @@ public class ChangeNotesTest {
otherUser = userFactory.create(ou.getId());
}
private void setMillisProvider() {
private void setTimeForTesting() {
systemTimeZone = System.setProperty("user.timezone", "US/Eastern");
clockStepMs = MILLISECONDS.convert(1, SECONDS);
final AtomicLong clockMs = new AtomicLong(
MILLISECONDS.convert(ChangeUtil.SORT_KEY_EPOCH_MINS, MINUTES)
+ MILLISECONDS.convert(60, DAYS));
new DateTime(2009, 9, 30, 17, 0, 0).getMillis());
DateTimeUtils.setCurrentMillisProvider(new MillisProvider() {
@Override
@@ -174,8 +173,9 @@ public class ChangeNotesTest {
}
@After
public void resetMillisProvider() {
public void resetTime() {
DateTimeUtils.setCurrentMillisSystem();
System.setProperty("user.timezone", systemTimeZone);
}
@Test
@@ -207,7 +207,7 @@ public class ChangeNotesTest {
assertEquals("1@gerrit", author.getEmailAddress());
assertEquals(new Date(c.getCreatedOn().getTime() + 1000),
author.getWhen());
assertEquals(TimeZone.getTimeZone("GMT-8:00"), author.getTimeZone());
assertEquals(TimeZone.getTimeZone("GMT-7:00"), author.getTimeZone());
PersonIdent committer = commit.getCommitterIdent();
assertEquals("Gerrit Server", committer.getName());

125
gerrit-server/src/test/java/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java
View File

@@ -432,51 +432,104 @@ public abstract class AbstractQueryChangesTest {
}
@Test
public void pagination() throws Exception {
public void start() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
List<Change> changes = Lists.newArrayList();
for (int i = 0; i < 5; i++) {
for (int i = 0; i < 2; i++) {
changes.add(newChange(repo, null, null, null, null).insert());
}
QueryChanges q;
List<ChangeInfo> results;
results = query("status:new");
assertEquals(2, results.size());
assertResultEquals(changes.get(1), results.get(0));
assertResultEquals(changes.get(0), results.get(1));
q = newQuery("status:new");
q.setStart(1);
results = query(q);
assertEquals(1, results.size());
assertResultEquals(changes.get(0), results.get(0));
q = newQuery("status:new");
q.setStart(2);
results = query(q);
assertEquals(0, results.size());
q = newQuery("status:new");
q.setStart(3);
results = query(q);
assertEquals(0, results.size());
}
@Test
public void startWithLimit() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
List<Change> changes = Lists.newArrayList();
for (int i = 0; i < 3; i++) {
changes.add(newChange(repo, null, null, null, null).insert());
}
// Page forward and back through 3 pages of results.
QueryChanges q;
List<ChangeInfo> results;
results = query("status:new limit:2");
assertEquals(2, results.size());
assertResultEquals(changes.get(4), results.get(0));
assertResultEquals(changes.get(3), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyBefore(results.get(1)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(2), results.get(0));
assertResultEquals(changes.get(1), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyBefore(results.get(1)._sortkey);
q.setStart(1);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(1), results.get(0));
assertResultEquals(changes.get(0), results.get(1));
q = newQuery("status:new limit:2");
q.setStart(2);
results = query(q);
assertEquals(1, results.size());
assertResultEquals(changes.get(0), results.get(0));
q = newQuery("status:new limit:2");
q.setSortKeyAfter(results.get(0)._sortkey);
q.setStart(3);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(2), results.get(0));
assertResultEquals(changes.get(1), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyAfter(results.get(0)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(4), results.get(0));
assertResultEquals(changes.get(3), results.get(1));
assertEquals(0, results.size());
}
@Test
public void sortKeyWithMinuteResolution() throws Exception {
public void updateOrder() throws Exception {
clockStepMs = MILLISECONDS.convert(2, MINUTES);
TestRepository<InMemoryRepository> repo = createProject("repo");
List<ChangeInserter> inserters = Lists.newArrayList();
List<Change> changes = Lists.newArrayList();
for (int i = 0; i < 5; i++) {
inserters.add(newChange(repo, null, null, null, null));
changes.add(inserters.get(i).insert());
}
for (int i : ImmutableList.of(2, 0, 1, 4, 3)) {
ReviewInput input = new ReviewInput();
input.message = "modifying " + i;
postReview.apply(
new RevisionResource(
this.changes.parse(changes.get(i).getId()),
inserters.get(i).getPatchSet()),
input);
changes.set(i, db.changes().get(changes.get(i).getId()));
}
List<ChangeInfo> results = query("status:new");
assertEquals(5, results.size());
assertResultEquals(changes.get(3), results.get(0));
assertResultEquals(changes.get(4), results.get(1));
assertResultEquals(changes.get(1), results.get(2));
assertResultEquals(changes.get(0), results.get(3));
assertResultEquals(changes.get(2), results.get(4));
}
@Test
public void updatedOrderWithMinuteResolution() throws Exception {
clockStepMs = MILLISECONDS.convert(2, MINUTES);
TestRepository<InMemoryRepository> repo = createProject("repo");
ChangeInserter ins1 = newChange(repo, null, null, null, null);
@@ -509,7 +562,7 @@ public abstract class AbstractQueryChangesTest {
}
@Test
public void sortKeyWithSubMinuteResolution() throws Exception {
public void updatedOrderWithSubMinuteResolution() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
ChangeInserter ins1 = newChange(repo, null, null, null, null);
Change change1 = ins1.insert();
@@ -535,23 +588,9 @@ public abstract class AbstractQueryChangesTest {
results = query("status:new");
assertEquals(2, results.size());
// Same order as before change1 was modified.
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
}
@Test
public void sortKeyBreaksTiesOnChangeId() throws Exception {
clockStepMs = 0;
TestRepository<InMemoryRepository> repo = createProject("repo");
Change change1 = newChange(repo, null, null, null, null).insert();
Change change2 = newChange(repo, null, null, null, null).insert();
assertEquals(change1.getLastUpdatedOn(), change2.getLastUpdatedOn());
List<ChangeInfo> results = query("status:new");
assertEquals(2, results.size());
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
// change1 moved to the top.
assertResultEquals(change1, results.get(0));
assertResultEquals(change2, results.get(1));
}
@Test
@@ -564,7 +603,7 @@ public abstract class AbstractQueryChangesTest {
newChange(repo, null, null, user2, null).insert();
}
assertResultEquals(change, queryOne("status:new ownerin:Administrators"));
//assertResultEquals(change, queryOne("status:new ownerin:Administrators"));
assertResultEquals(change,
queryOne("status:new ownerin:Administrators limit:2"));
}
@@ -877,7 +916,7 @@ public abstract class AbstractQueryChangesTest {
return results.get(0);
}
private static long lastUpdatedMs(Change c) {
protected static long lastUpdatedMs(Change c) {
return c.getLastUpdatedOn().getTime();
}
}

145
gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV7Test.java Normal file
View File

@@ -0,0 +1,145 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.change;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import com.google.common.collect.Lists;
import com.google.gerrit.extensions.api.changes.ReviewInput;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.change.ChangeInserter;
import com.google.gerrit.server.change.ChangeJson.ChangeInfo;
import com.google.gerrit.server.change.RevisionResource;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Config;
import org.junit.Test;
import java.util.List;
public class LuceneQueryChangesV7Test extends AbstractQueryChangesTest {
protected Injector createInjector() {
Config cfg = InMemoryModule.newDefaultConfig();
cfg.setInt("index", "lucene", "testVersion", 7);
return Guice.createInjector(new InMemoryModule(cfg));
}
@Test
public void pagination() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
List<Change> changes = Lists.newArrayList();
for (int i = 0; i < 5; i++) {
changes.add(newChange(repo, null, null, null, null).insert());
}
// Page forward and back through 3 pages of results.
QueryChanges q;
List<ChangeInfo> results;
results = query("status:new limit:2");
assertEquals(2, results.size());
assertResultEquals(changes.get(4), results.get(0));
assertResultEquals(changes.get(3), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyBefore(results.get(1)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(2), results.get(0));
assertResultEquals(changes.get(1), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyBefore(results.get(1)._sortkey);
results = query(q);
assertEquals(1, results.size());
assertResultEquals(changes.get(0), results.get(0));
q = newQuery("status:new limit:2");
q.setSortKeyAfter(results.get(0)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(2), results.get(0));
assertResultEquals(changes.get(1), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyAfter(results.get(0)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(4), results.get(0));
assertResultEquals(changes.get(3), results.get(1));
}
@Override
@Test
public void updatedOrderWithSubMinuteResolution() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
ChangeInserter ins1 = newChange(repo, null, null, null, null);
Change change1 = ins1.insert();
Change change2 = newChange(repo, null, null, null, null).insert();
assertTrue(lastUpdatedMs(change1) < lastUpdatedMs(change2));
List<ChangeInfo> results;
results = query("status:new");
assertEquals(2, results.size());
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
ReviewInput input = new ReviewInput();
input.message = "toplevel";
postReview.apply(new RevisionResource(
changes.parse(change1.getId()), ins1.getPatchSet()), input);
change1 = db.changes().get(change1.getId());
assertTrue(lastUpdatedMs(change1) > lastUpdatedMs(change2));
assertTrue(lastUpdatedMs(change1) - lastUpdatedMs(change2)
< MILLISECONDS.convert(1, MINUTES));
results = query("status:new");
assertEquals(2, results.size());
// Same order as before change1 was modified.
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
}
@Test
public void sortKeyBreaksTiesOnChangeId() throws Exception {
clockStepMs = 0;
TestRepository<InMemoryRepository> repo = createProject("repo");
ChangeInserter ins1 = newChange(repo, null, null, null, null);
Change change1 = ins1.insert();
Change change2 = newChange(repo, null, null, null, null).insert();
ReviewInput input = new ReviewInput();
input.message = "toplevel";
postReview.apply(new RevisionResource(
changes.parse(change1.getId()), ins1.getPatchSet()), input);
change1 = db.changes().get(change1.getId());
assertEquals(change1.getLastUpdatedOn(), change2.getLastUpdatedOn());
List<ChangeInfo> results = query("status:new");
assertEquals(2, results.size());
// Updated at the same time, 2 > 1.
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
}
}

49
gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java
View File

@@ -20,6 +20,7 @@ import static com.google.gerrit.solr.IndexVersionCheck.SCHEMA_VERSIONS;
import static com.google.gerrit.solr.IndexVersionCheck.solrIndexConfig;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.gerrit.extensions.events.LifecycleListener;
@@ -41,6 +42,7 @@ import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gerrit.server.query.change.ChangeQueryBuilder;
import com.google.gerrit.server.query.change.SortKeyPredicate;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Provider;
@@ -50,6 +52,7 @@ import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Version;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.SortClause;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
@@ -210,7 +213,7 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p, int limit)
public ChangeDataSource getSource(Predicate<ChangeData> p, int start, int limit)
throws QueryParseException {
Set<Change.Status> statuses = IndexRewriteImpl.getPossibleStatus(p);
List<SolrServer> indexes = Lists.newArrayListWithCapacity(2);
@@ -220,8 +223,24 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
if (!Sets.intersection(statuses, CLOSED_STATUSES).isEmpty()) {
indexes.add(closedIndex);
}
return new QuerySource(indexes, queryBuilder.toQuery(p), limit,
ChangeQueryBuilder.hasNonTrivialSortKeyAfter(schema, p));
return new QuerySource(indexes, queryBuilder.toQuery(p), start, limit,
getSorts(schema, p));
}
@SuppressWarnings("deprecation")
private static List<SortClause> getSorts(Schema<ChangeData> schema,
Predicate<ChangeData> p) {
if (SortKeyPredicate.hasSortKeyField(schema)) {
boolean reverse = ChangeQueryBuilder.hasNonTrivialSortKeyAfter(schema, p);
return ImmutableList.of(new SortClause(ChangeField.SORTKEY.getName(),
!reverse ? SolrQuery.ORDER.desc : SolrQuery.ORDER.asc));
} else {
return ImmutableList.of(
new SortClause(
ChangeField.UPDATED.getName(), SolrQuery.ORDER.desc),
new SortClause(
ChangeField.LEGACY_ID.getName(), SolrQuery.ORDER.desc));
}
}
private void commit(SolrServer server) throws IOException {
@@ -236,17 +255,18 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
private final List<SolrServer> indexes;
private final SolrQuery query;
public QuerySource(List<SolrServer> indexes, Query q, int limit,
boolean reverse) {
public QuerySource(List<SolrServer> indexes, Query q, int start, int limit,
List<SortClause> sorts) {
this.indexes = indexes;
query = new SolrQuery(q.toString());
query.setParam("shards.tolerant", true);
query.setParam("rows", Integer.toString(limit));
if (start != 0) {
query.setParam("start", Integer.toString(start));
}
query.setFields(ID_FIELD);
query.setSort(
ChangeField.SORTKEY.getName(),
!reverse ? SolrQuery.ORDER.desc : SolrQuery.ORDER.asc);
query.setSorts(sorts);
}
@Override
@@ -329,8 +349,17 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
doc.addField(name, (Long) value);
}
} else if (type == FieldType.TIMESTAMP) {
for (Object v : values.getValues()) {
doc.addField(name, QueryBuilder.toIndexTime((Timestamp) v));
@SuppressWarnings("deprecation")
boolean legacy = values.getField() == ChangeField.LEGACY_UPDATED;
if (legacy) {
for (Object value : values.getValues()) {
int t = queryBuilder.toIndexTimeInMinutes((Timestamp) value);
doc.addField(name, t);
}
} else {
for (Object value : values.getValues()) {
doc.addField(name, ((Timestamp) value).getTime());
}
}
} else if (type == FieldType.EXACT
|| type == FieldType.PREFIX