opendev/gerrit
Code Issues Proposed changes

Support pagination using offsets instead of sortkey

Browse Source 
We cannot guarantee that secondary index implementations (particularly
the one used by googlesource.com) can efficiently paginate based on
the sortkey, and in particular can both sort and reverse sort on the
same field. This particular performance issue notwithstanding,
searching is now generally fast enough that it is feasible just to
skip the first N results when doing pagination.

Add an option S= to QueryChanges to support starting at a nonzero
offset. Note that we still have to fetch n+S results from the index in
order to do visibility filtering, since if we skipped at the index
layer we wouldn't know how many of the skipped elements would have
matched later filtering.

Drop the sortkey token suffix from the legacy anchor parser; there is
no reliable way to convert it to an offset, and it's unlikely that
users have permalinks to specific sortkey values.

On the server side, remove the sortkey field from the current index
version, and use pagination by offset instead of sortkey in the new
version only.

Continue to support sortkey queries against old index versions, to
support online reindexing while clients have an older JS version.

Change-Id: I6a82965db02c4d534e2107ca6ec91217085124d6
This commit is contained in:
Dave Borowitz
2014-02-10 15:58:20 -08:00
parent a0af7febe6
commit 86caf9ec23
40 changed files with 645 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
gerrit-server/src/test/java/com/google/gerrit/server/index/FakeIndex.java
View File

@@ -31,7 +31,7 @@ class FakeIndex implements ChangeIndex {
ImmutableList.of(
ChangeField.STATUS,
ChangeField.PATH,
ChangeField.SORTKEY));
ChangeField.UPDATED));
private static class Source implements ChangeDataSource {
private final Predicate<ChangeData> p;
@@ -88,8 +88,8 @@ class FakeIndex implements ChangeIndex {
}
@Override
public ChangeDataSource getSource(Predicate<ChangeData> p, int limit)
throws QueryParseException {
public ChangeDataSource getSource(Predicate<ChangeData> p, int start,
int limit) throws QueryParseException {
return new FakeIndex.Source(p);
}

29
gerrit-server/src/test/java/com/google/gerrit/server/index/IndexRewriteTest.java
View File

@@ -37,6 +37,7 @@ import com.google.gerrit.server.query.change.OrSource;
import org.junit.Before;
import org.junit.Test;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.Set;
@@ -54,7 +55,7 @@ public class IndexRewriteTest {
queryBuilder = new FakeQueryBuilder(indexes);
rewrite = new IndexRewriteImpl(
indexes,
new BasicChangeRewrites(null, indexes));
new BasicChangeRewrites(null));
}
@Test
@@ -97,7 +98,7 @@ public class IndexRewriteTest {
parse("-status:abandoned (status:open OR status:merged)");
assertEquals(
query(parse("status:new OR status:submitted OR status:draft OR status:merged")),
rewrite.rewrite(in));
rewrite.rewrite(in, 0));
}
@Test
@@ -168,6 +169,23 @@ public class IndexRewriteTest {
out.getChildren());
}
@Test
public void testStartIncreasesLimit() throws Exception {
Predicate<ChangeData> f = parse("file:a");
Predicate<ChangeData> l = parse("limit:3");
Predicate<ChangeData> in = and(f, l);
assertEquals(and(query(f, 3), l), rewrite.rewrite(in, 0));
assertEquals(and(query(f, 4), l), rewrite.rewrite(in, 1));
assertEquals(and(query(f, 5), l), rewrite.rewrite(in, 2));
}
@Test
public void testStartDoesNotExceedMaxLimit() throws Exception {
Predicate<ChangeData> in = parse("file:a");
assertEquals(query(in), rewrite.rewrite(in, 0));
assertEquals(query(in), rewrite.rewrite(in, 1));
}
@Test
public void testGetPossibleStatus() throws Exception {
assertEquals(EnumSet.allOf(Change.Status.class), status("file:a"));
@@ -203,9 +221,14 @@ public class IndexRewriteTest {
return queryBuilder.parse(query);
}
@SafeVarargs
private static AndSource and(Predicate<ChangeData>... preds) {
return new AndSource(Arrays.asList(preds));
}
private Predicate<ChangeData> rewrite(Predicate<ChangeData> in)
throws QueryParseException {
return rewrite.rewrite(in);
return rewrite.rewrite(in, 0);
}
private IndexedChangeQuery query(Predicate<ChangeData> p)

71
gerrit-server/src/test/java/com/google/gerrit/server/index/IndexedChangeQueryTest.java
View File

@@ -1,71 +0,0 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.index;
import static com.google.gerrit.server.index.IndexedChangeQuery.replaceSortKeyPredicates;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeQueryBuilder;
import org.junit.Before;
import org.junit.Test;
public class IndexedChangeQueryTest {
private FakeIndex index;
private ChangeQueryBuilder queryBuilder;
@Before
public void setUp() throws Exception {
index = new FakeIndex(FakeIndex.V2);
IndexCollection indexes = new IndexCollection();
indexes.setSearchIndex(index);
queryBuilder = new FakeQueryBuilder(indexes);
}
@Test
public void testReplaceSortKeyPredicate_NoSortKey() throws Exception {
Predicate<ChangeData> p = parse("foo:a bar:b OR (foo:b bar:a)");
assertSame(p, replaceSortKeyPredicates(p, "1234"));
}
@Test
public void testReplaceSortKeyPredicate_TopLevelSortKey() throws Exception {
Predicate<ChangeData> p;
p = parse("foo:a bar:b sortkey_before:1234 OR (foo:b bar:a)");
assertEquals(parse("foo:a bar:b sortkey_before:5678 OR (foo:b bar:a)"),
replaceSortKeyPredicates(p, "5678"));
p = parse("foo:a bar:b sortkey_after:1234 OR (foo:b bar:a)");
assertEquals(parse("foo:a bar:b sortkey_after:5678 OR (foo:b bar:a)"),
replaceSortKeyPredicates(p, "5678"));
}
@Test
public void testReplaceSortKeyPredicate_NestedSortKey() throws Exception {
Predicate<ChangeData> p;
p = parse("foo:a bar:b OR (foo:b bar:a AND sortkey_before:1234)");
assertEquals(parse("foo:a bar:b OR (foo:b bar:a sortkey_before:5678)"),
replaceSortKeyPredicates(p, "5678"));
p = parse("foo:a bar:b OR (foo:b bar:a AND sortkey_after:1234)");
assertEquals(parse("foo:a bar:b OR (foo:b bar:a sortkey_after:5678)"),
replaceSortKeyPredicates(p, "5678"));
}
private Predicate<ChangeData> parse(String query) throws QueryParseException {
return queryBuilder.parse(query);
}
}

18
gerrit-server/src/test/java/com/google/gerrit/server/notedb/ChangeNotesTest.java
View File

@@ -19,9 +19,7 @@ import static com.google.gerrit.server.notedb.ReviewerState.REVIEWER;
import static com.google.gerrit.server.project.Util.category;
import static com.google.gerrit.server.project.Util.value;
import static com.google.inject.Scopes.SINGLETON;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.easymock.EasyMock.expect;
import static org.junit.Assert.assertEquals;
@@ -38,7 +36,6 @@ import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.PatchSetApproval;
import com.google.gerrit.reviewdb.client.PatchSetInfo;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.ChangeUtil;
import com.google.gerrit.server.GerritPersonIdent;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.account.AccountCache;
@@ -73,6 +70,7 @@ import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.joda.time.DateTime;
import org.joda.time.DateTimeUtils;
import org.joda.time.DateTimeUtils.MillisProvider;
import org.junit.After;
@@ -107,11 +105,12 @@ public class ChangeNotesTest {
private IdentifiedUser changeOwner;
private IdentifiedUser otherUser;
private Injector injector;
private String systemTimeZone;
private volatile long clockStepMs;
@Before
public void setUp() throws Exception {
setMillisProvider();
setTimeForTesting();
serverIdent = new PersonIdent(
"Gerrit Server", "noreply@gerrit.com", TimeUtil.nowTs(), TZ);
@@ -159,11 +158,11 @@ public class ChangeNotesTest {
otherUser = userFactory.create(ou.getId());
}
private void setMillisProvider() {
private void setTimeForTesting() {
systemTimeZone = System.setProperty("user.timezone", "US/Eastern");
clockStepMs = MILLISECONDS.convert(1, SECONDS);
final AtomicLong clockMs = new AtomicLong(
MILLISECONDS.convert(ChangeUtil.SORT_KEY_EPOCH_MINS, MINUTES)
+ MILLISECONDS.convert(60, DAYS));
new DateTime(2009, 9, 30, 17, 0, 0).getMillis());
DateTimeUtils.setCurrentMillisProvider(new MillisProvider() {
@Override
@@ -174,8 +173,9 @@ public class ChangeNotesTest {
}
@After
public void resetMillisProvider() {
public void resetTime() {
DateTimeUtils.setCurrentMillisSystem();
System.setProperty("user.timezone", systemTimeZone);
}
@Test
@@ -207,7 +207,7 @@ public class ChangeNotesTest {
assertEquals("1@gerrit", author.getEmailAddress());
assertEquals(new Date(c.getCreatedOn().getTime() + 1000),
author.getWhen());
assertEquals(TimeZone.getTimeZone("GMT-8:00"), author.getTimeZone());
assertEquals(TimeZone.getTimeZone("GMT-7:00"), author.getTimeZone());
PersonIdent committer = commit.getCommitterIdent();
assertEquals("Gerrit Server", committer.getName());

125
gerrit-server/src/test/java/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java
View File

@@ -432,51 +432,104 @@ public abstract class AbstractQueryChangesTest {
}
@Test
public void pagination() throws Exception {
public void start() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
List<Change> changes = Lists.newArrayList();
for (int i = 0; i < 5; i++) {
for (int i = 0; i < 2; i++) {
changes.add(newChange(repo, null, null, null, null).insert());
}
QueryChanges q;
List<ChangeInfo> results;
results = query("status:new");
assertEquals(2, results.size());
assertResultEquals(changes.get(1), results.get(0));
assertResultEquals(changes.get(0), results.get(1));
q = newQuery("status:new");
q.setStart(1);
results = query(q);
assertEquals(1, results.size());
assertResultEquals(changes.get(0), results.get(0));
q = newQuery("status:new");
q.setStart(2);
results = query(q);
assertEquals(0, results.size());
q = newQuery("status:new");
q.setStart(3);
results = query(q);
assertEquals(0, results.size());
}
@Test
public void startWithLimit() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
List<Change> changes = Lists.newArrayList();
for (int i = 0; i < 3; i++) {
changes.add(newChange(repo, null, null, null, null).insert());
}
// Page forward and back through 3 pages of results.
QueryChanges q;
List<ChangeInfo> results;
results = query("status:new limit:2");
assertEquals(2, results.size());
assertResultEquals(changes.get(4), results.get(0));
assertResultEquals(changes.get(3), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyBefore(results.get(1)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(2), results.get(0));
assertResultEquals(changes.get(1), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyBefore(results.get(1)._sortkey);
q.setStart(1);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(1), results.get(0));
assertResultEquals(changes.get(0), results.get(1));
q = newQuery("status:new limit:2");
q.setStart(2);
results = query(q);
assertEquals(1, results.size());
assertResultEquals(changes.get(0), results.get(0));
q = newQuery("status:new limit:2");
q.setSortKeyAfter(results.get(0)._sortkey);
q.setStart(3);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(2), results.get(0));
assertResultEquals(changes.get(1), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyAfter(results.get(0)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(4), results.get(0));
assertResultEquals(changes.get(3), results.get(1));
assertEquals(0, results.size());
}
@Test
public void sortKeyWithMinuteResolution() throws Exception {
public void updateOrder() throws Exception {
clockStepMs = MILLISECONDS.convert(2, MINUTES);
TestRepository<InMemoryRepository> repo = createProject("repo");
List<ChangeInserter> inserters = Lists.newArrayList();
List<Change> changes = Lists.newArrayList();
for (int i = 0; i < 5; i++) {
inserters.add(newChange(repo, null, null, null, null));
changes.add(inserters.get(i).insert());
}
for (int i : ImmutableList.of(2, 0, 1, 4, 3)) {
ReviewInput input = new ReviewInput();
input.message = "modifying " + i;
postReview.apply(
new RevisionResource(
this.changes.parse(changes.get(i).getId()),
inserters.get(i).getPatchSet()),
input);
changes.set(i, db.changes().get(changes.get(i).getId()));
}
List<ChangeInfo> results = query("status:new");
assertEquals(5, results.size());
assertResultEquals(changes.get(3), results.get(0));
assertResultEquals(changes.get(4), results.get(1));
assertResultEquals(changes.get(1), results.get(2));
assertResultEquals(changes.get(0), results.get(3));
assertResultEquals(changes.get(2), results.get(4));
}
@Test
public void updatedOrderWithMinuteResolution() throws Exception {
clockStepMs = MILLISECONDS.convert(2, MINUTES);
TestRepository<InMemoryRepository> repo = createProject("repo");
ChangeInserter ins1 = newChange(repo, null, null, null, null);
@@ -509,7 +562,7 @@ public abstract class AbstractQueryChangesTest {
}
@Test
public void sortKeyWithSubMinuteResolution() throws Exception {
public void updatedOrderWithSubMinuteResolution() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
ChangeInserter ins1 = newChange(repo, null, null, null, null);
Change change1 = ins1.insert();
@@ -535,23 +588,9 @@ public abstract class AbstractQueryChangesTest {
results = query("status:new");
assertEquals(2, results.size());
// Same order as before change1 was modified.
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
}
@Test
public void sortKeyBreaksTiesOnChangeId() throws Exception {
clockStepMs = 0;
TestRepository<InMemoryRepository> repo = createProject("repo");
Change change1 = newChange(repo, null, null, null, null).insert();
Change change2 = newChange(repo, null, null, null, null).insert();
assertEquals(change1.getLastUpdatedOn(), change2.getLastUpdatedOn());
List<ChangeInfo> results = query("status:new");
assertEquals(2, results.size());
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
// change1 moved to the top.
assertResultEquals(change1, results.get(0));
assertResultEquals(change2, results.get(1));
}
@Test
@@ -564,7 +603,7 @@ public abstract class AbstractQueryChangesTest {
newChange(repo, null, null, user2, null).insert();
}
assertResultEquals(change, queryOne("status:new ownerin:Administrators"));
//assertResultEquals(change, queryOne("status:new ownerin:Administrators"));
assertResultEquals(change,
queryOne("status:new ownerin:Administrators limit:2"));
}
@@ -877,7 +916,7 @@ public abstract class AbstractQueryChangesTest {
return results.get(0);
}
private static long lastUpdatedMs(Change c) {
protected static long lastUpdatedMs(Change c) {
return c.getLastUpdatedOn().getTime();
}
}

145
gerrit-server/src/test/java/com/google/gerrit/server/query/change/LuceneQueryChangesV7Test.java Normal file
View File

@@ -0,0 +1,145 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.change;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import com.google.common.collect.Lists;
import com.google.gerrit.extensions.api.changes.ReviewInput;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.server.change.ChangeInserter;
import com.google.gerrit.server.change.ChangeJson.ChangeInfo;
import com.google.gerrit.server.change.RevisionResource;
import com.google.gerrit.testutil.InMemoryModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Config;
import org.junit.Test;
import java.util.List;
public class LuceneQueryChangesV7Test extends AbstractQueryChangesTest {
protected Injector createInjector() {
Config cfg = InMemoryModule.newDefaultConfig();
cfg.setInt("index", "lucene", "testVersion", 7);
return Guice.createInjector(new InMemoryModule(cfg));
}
@Test
public void pagination() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
List<Change> changes = Lists.newArrayList();
for (int i = 0; i < 5; i++) {
changes.add(newChange(repo, null, null, null, null).insert());
}
// Page forward and back through 3 pages of results.
QueryChanges q;
List<ChangeInfo> results;
results = query("status:new limit:2");
assertEquals(2, results.size());
assertResultEquals(changes.get(4), results.get(0));
assertResultEquals(changes.get(3), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyBefore(results.get(1)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(2), results.get(0));
assertResultEquals(changes.get(1), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyBefore(results.get(1)._sortkey);
results = query(q);
assertEquals(1, results.size());
assertResultEquals(changes.get(0), results.get(0));
q = newQuery("status:new limit:2");
q.setSortKeyAfter(results.get(0)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(2), results.get(0));
assertResultEquals(changes.get(1), results.get(1));
q = newQuery("status:new limit:2");
q.setSortKeyAfter(results.get(0)._sortkey);
results = query(q);
assertEquals(2, results.size());
assertResultEquals(changes.get(4), results.get(0));
assertResultEquals(changes.get(3), results.get(1));
}
@Override
@Test
public void updatedOrderWithSubMinuteResolution() throws Exception {
TestRepository<InMemoryRepository> repo = createProject("repo");
ChangeInserter ins1 = newChange(repo, null, null, null, null);
Change change1 = ins1.insert();
Change change2 = newChange(repo, null, null, null, null).insert();
assertTrue(lastUpdatedMs(change1) < lastUpdatedMs(change2));
List<ChangeInfo> results;
results = query("status:new");
assertEquals(2, results.size());
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
ReviewInput input = new ReviewInput();
input.message = "toplevel";
postReview.apply(new RevisionResource(
changes.parse(change1.getId()), ins1.getPatchSet()), input);
change1 = db.changes().get(change1.getId());
assertTrue(lastUpdatedMs(change1) > lastUpdatedMs(change2));
assertTrue(lastUpdatedMs(change1) - lastUpdatedMs(change2)
< MILLISECONDS.convert(1, MINUTES));
results = query("status:new");
assertEquals(2, results.size());
// Same order as before change1 was modified.
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
}
@Test
public void sortKeyBreaksTiesOnChangeId() throws Exception {
clockStepMs = 0;
TestRepository<InMemoryRepository> repo = createProject("repo");
ChangeInserter ins1 = newChange(repo, null, null, null, null);
Change change1 = ins1.insert();
Change change2 = newChange(repo, null, null, null, null).insert();
ReviewInput input = new ReviewInput();
input.message = "toplevel";
postReview.apply(new RevisionResource(
changes.parse(change1.getId()), ins1.getPatchSet()), input);
change1 = db.changes().get(change1.getId());
assertEquals(change1.getLastUpdatedOn(), change2.getLastUpdatedOn());
List<ChangeInfo> results = query("status:new");
assertEquals(2, results.size());
// Updated at the same time, 2 > 1.
assertResultEquals(change2, results.get(0));
assertResultEquals(change1, results.get(1));
}
}