Files
gerrit/javatests/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java
Edwin Kempin 04b208f14f Revert "Temporarily disable reindexIfStale for accounts"
The account staleness checker was disabled because of failures in the
Elasticsearch tests that looked like [1]. The investigation of this
issue showed that this was not caused by the account staleness checker
but was an issue that could already occur before. For some reason the
issue is hit more often when the staleness checker is enabled. Change
I7b20a0cee6 fixed this partly, but the problem can still occur. In
general the unstable Elasticsearch tests should not prevent us from
enabling unrelated features like the staleness checker for accounts.
The CI already excludes the Elasticsearch tests since they are flaky.
Fixing the flakyness of the Elasticsearch tests is unrelated to the
account staleness checker and should be done in separate changes.
However to not make the Elasticsearch tests any more flaky disable the
staleness checker for all Elasticsearch tests.

However a real issue with enabling the account staleness checker was
that it made some account tests flaky. Some account tests verify the
number of times that an account is indexed. With the account staleness
checker these tests got flaky because the staleness checker caused extra
reindex events. This may happen if an account is reindexed several times
in a row since the staleness check is done asynchronously, e.g. if an
account is reindexed twice the following happens:

1. TestThread: update account to state A, index and trigger reindex if
   stale
2. BatchExecutorThread: auto reindex if stale, triggered by 1.
3. TestThread: update account to state B, index and trigger reindex if
   stale
4. BatchExecutorThread: auto reindex if stale, triggered by 3.

Since 2 and 3 are executed in parallel it can happen that 2 reads state
B from NoteDb and state A from index and hence detects the account as
stale which triggers another reindex. This is fine, it's an extra
reindex event but the logic ensures that in the end the account is never
stale in the index.

However since this is confusing the account tests the staleness checker
is now explicitly disabled for all account tests.

There was also some flakiness observed when we expected only a single
index event [2] but I wasn't able to reproduce this. Disabling the
staleness checker for all account tests makes sure that we don't get
this flakiness either.

This reverts commit b316186826.

[1]
[accounts_0007] IndexAlreadyExistsException[already exists]
  at org.elasticsearch.cluster.metadata.MetaDataCreateIndexService.validateIndexName(MetaDataCreateIndexService.java:136)
  at org.elasticsearch.cluster.metadata.MetaDataCreateIndexService.validate(MetaDataCreateIndexService.java:431)
  at org.elasticsearch.cluster.metadata.MetaDataCreateIndexService.access$100(MetaDataCreateIndexService.java:95)
  at org.elasticsearch.cluster.metadata.MetaDataCreateIndexService$1.execute(MetaDataCreateIndexService.java:190)
  at org.elasticsearch.cluster.ClusterStateUpdateTask.execute(ClusterStateUpdateTask.java:45)
  at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:480)
  at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  at java.lang.Thread.run(Thread.java:745)

[2] https://gerrit-ci.gerritforge.com/job/Gerrit-verifier-bazel/35988/consoleText

Change-Id: Ib0c69dd6805b2624679afa10d2ef2fd89dc5f8be
Signed-off-by: Edwin Kempin <ekempin@google.com>
2017-11-24 15:11:50 +00:00

370 lines
12 KiB
Java

// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.query.project;
import static com.google.common.truth.Truth.assertThat;
import static java.util.stream.Collectors.toList;
import com.google.common.base.CharMatcher;
import com.google.gerrit.extensions.api.GerritApi;
import com.google.gerrit.extensions.api.projects.ProjectInput;
import com.google.gerrit.extensions.api.projects.Projects.QueryRequest;
import com.google.gerrit.extensions.common.AccountInfo;
import com.google.gerrit.extensions.common.ProjectInfo;
import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.lifecycle.LifecycleManager;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.AnonymousUser;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.account.AccountCache;
import com.google.gerrit.server.account.AccountManager;
import com.google.gerrit.server.account.Accounts;
import com.google.gerrit.server.account.AccountsUpdate;
import com.google.gerrit.server.account.AuthRequest;
import com.google.gerrit.server.config.AllProjectsName;
import com.google.gerrit.server.query.account.InternalAccountQuery;
import com.google.gerrit.server.schema.SchemaCreator;
import com.google.gerrit.server.util.ManualRequestContext;
import com.google.gerrit.server.util.OneOffRequestContext;
import com.google.gerrit.server.util.RequestContext;
import com.google.gerrit.server.util.ThreadLocalRequestContext;
import com.google.gerrit.testing.GerritServerTests;
import com.google.gerrit.testing.InMemoryDatabase;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Provider;
import com.google.inject.util.Providers;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@Ignore
public abstract class AbstractQueryProjectsTest extends GerritServerTests {
@Inject protected Accounts accounts;
@Inject protected AccountsUpdate.Server accountsUpdate;
@Inject protected AccountCache accountCache;
@Inject protected AccountManager accountManager;
@Inject protected GerritApi gApi;
@Inject protected IdentifiedUser.GenericFactory userFactory;
@Inject private Provider<AnonymousUser> anonymousUser;
@Inject protected InMemoryDatabase schemaFactory;
@Inject protected SchemaCreator schemaCreator;
@Inject protected ThreadLocalRequestContext requestContext;
@Inject protected OneOffRequestContext oneOffRequestContext;
@Inject protected InternalAccountQuery internalAccountQuery;
@Inject protected AllProjectsName allProjects;
protected LifecycleManager lifecycle;
protected Injector injector;
protected ReviewDb db;
protected AccountInfo currentUserInfo;
protected CurrentUser user;
protected abstract Injector createInjector();
@Before
public void setUpInjector() throws Exception {
lifecycle = new LifecycleManager();
injector = createInjector();
lifecycle.add(injector);
injector.injectMembers(this);
lifecycle.start();
setUpDatabase();
}
@After
public void cleanUp() {
lifecycle.stop();
db.close();
}
protected void setUpDatabase() throws Exception {
db = schemaFactory.open();
schemaCreator.create(db);
Account.Id userId = createAccount("user", "User", "user@example.com", true);
user = userFactory.create(userId);
requestContext.setContext(newRequestContext(userId));
currentUserInfo = gApi.accounts().id(userId.get()).get();
}
protected RequestContext newRequestContext(Account.Id requestUserId) {
final CurrentUser requestUser = userFactory.create(requestUserId);
return new RequestContext() {
@Override
public CurrentUser getUser() {
return requestUser;
}
@Override
public Provider<ReviewDb> getReviewDbProvider() {
return Providers.of(db);
}
};
}
protected void setAnonymous() {
requestContext.setContext(
new RequestContext() {
@Override
public CurrentUser getUser() {
return anonymousUser.get();
}
@Override
public Provider<ReviewDb> getReviewDbProvider() {
return Providers.of(db);
}
});
}
@After
public void tearDownInjector() {
if (lifecycle != null) {
lifecycle.stop();
}
requestContext.setContext(null);
if (db != null) {
db.close();
}
InMemoryDatabase.drop(schemaFactory);
}
@Test
public void byName() throws Exception {
assertQuery("name:project");
assertQuery("name:non-existing");
ProjectInfo project = createProject(name("project"));
assertQuery("name:" + project.name, project);
// only exact match
ProjectInfo projectWithHyphen = createProject(name("project-with-hyphen"));
createProject(name("project-no-match-with-hyphen"));
assertQuery("name:" + projectWithHyphen.name, projectWithHyphen);
}
@Test
public void byInname() throws Exception {
String namePart = getSanitizedMethodName();
namePart = CharMatcher.is('_').removeFrom(namePart);
ProjectInfo project1 = createProject(name("project-" + namePart));
ProjectInfo project2 = createProject(name("project-" + namePart + "-2"));
ProjectInfo project3 = createProject(name("project-" + namePart + "3"));
assertQuery("inname:" + namePart, project1, project2, project3);
assertQuery("inname:" + namePart.toUpperCase(Locale.US), project1, project2, project3);
assertQuery("inname:" + namePart.toLowerCase(Locale.US), project1, project2, project3);
}
@Test
public void byDescription() throws Exception {
ProjectInfo project1 =
createProjectWithDescription(name("project1"), "This is a test project.");
ProjectInfo project2 = createProjectWithDescription(name("project2"), "ANOTHER TEST PROJECT.");
createProjectWithDescription(name("project3"), "Maintainers of project foo.");
assertQuery("description:test", project1, project2);
assertQuery("description:non-existing");
exception.expect(BadRequestException.class);
exception.expectMessage("description operator requires a value");
assertQuery("description:\"\"");
}
@Test
public void byDefaultField() throws Exception {
ProjectInfo project1 = createProject(name("foo-project"));
ProjectInfo project2 = createProject(name("project2"));
ProjectInfo project3 =
createProjectWithDescription(
name("project3"),
"decription that contains foo and the UUID of project2: " + project2.id);
assertQuery("non-existing");
assertQuery("foo", project1, project3);
assertQuery(project2.id, project2, project3);
}
@Test
public void withLimit() throws Exception {
ProjectInfo project1 = createProject(name("project1"));
ProjectInfo project2 = createProject(name("project2"));
ProjectInfo project3 = createProject(name("project3"));
String query =
"name:" + project1.name + " OR name:" + project2.name + " OR name:" + project3.name;
List<ProjectInfo> result = assertQuery(query, project1, project2, project3);
result = assertQuery(newQuery(query).withLimit(2), result.subList(0, 2));
}
@Test
public void withStart() throws Exception {
ProjectInfo project1 = createProject(name("project1"));
ProjectInfo project2 = createProject(name("project2"));
ProjectInfo project3 = createProject(name("project3"));
String query =
"name:" + project1.name + " OR name:" + project2.name + " OR name:" + project3.name;
List<ProjectInfo> result = assertQuery(query, project1, project2, project3);
assertQuery(newQuery(query).withStart(1), result.subList(1, 3));
}
@Test
public void asAnonymous() throws Exception {
ProjectInfo project = createProject(name("project"));
setAnonymous();
assertQuery("name:" + project.name);
}
private Account.Id createAccount(String username, String fullName, String email, boolean active)
throws Exception {
try (ManualRequestContext ctx = oneOffRequestContext.open()) {
Account.Id id = accountManager.authenticate(AuthRequest.forUser(username)).getAccountId();
if (email != null) {
accountManager.link(id, AuthRequest.forEmail(email));
}
accountsUpdate
.create()
.update(
id,
a -> {
a.setFullName(fullName);
a.setPreferredEmail(email);
a.setActive(active);
});
return id;
}
}
protected ProjectInfo createProject(String name) throws Exception {
ProjectInput in = new ProjectInput();
in.name = name;
return gApi.projects().create(in).get();
}
protected ProjectInfo createProjectWithDescription(String name, String description)
throws Exception {
ProjectInput in = new ProjectInput();
in.name = name;
in.description = description;
return gApi.projects().create(in).get();
}
protected ProjectInfo getProject(Project.NameKey nameKey) throws Exception {
return gApi.projects().name(nameKey.get()).get();
}
protected List<ProjectInfo> assertQuery(Object query, ProjectInfo... projects) throws Exception {
return assertQuery(newQuery(query), projects);
}
protected List<ProjectInfo> assertQuery(QueryRequest query, ProjectInfo... projects)
throws Exception {
return assertQuery(query, Arrays.asList(projects));
}
protected List<ProjectInfo> assertQuery(QueryRequest query, List<ProjectInfo> projects)
throws Exception {
List<ProjectInfo> result = query.get();
Iterable<String> names = names(result);
assertThat(names)
.named(format(query, result, projects))
.containsExactlyElementsIn(names(projects));
return result;
}
protected QueryRequest newQuery(Object query) {
return gApi.projects().query(query.toString());
}
protected String format(
QueryRequest query, List<ProjectInfo> actualProjects, List<ProjectInfo> expectedProjects) {
StringBuilder b = new StringBuilder();
b.append("query '").append(query.getQuery()).append("' with expected projects ");
b.append(format(expectedProjects));
b.append(" and result ");
b.append(format(actualProjects));
return b.toString();
}
protected String format(Iterable<ProjectInfo> projects) {
StringBuilder b = new StringBuilder();
b.append("[");
Iterator<ProjectInfo> it = projects.iterator();
while (it.hasNext()) {
ProjectInfo p = it.next();
b.append("{")
.append(p.id)
.append(", ")
.append("name=")
.append(p.name)
.append(", ")
.append("parent=")
.append(p.parent)
.append(", ")
.append("description=")
.append(p.description)
.append("}");
if (it.hasNext()) {
b.append(", ");
}
}
b.append("]");
return b.toString();
}
protected static Iterable<String> names(ProjectInfo... projects) {
return names(Arrays.asList(projects));
}
protected static Iterable<String> names(List<ProjectInfo> projects) {
return projects.stream().map(p -> p.name).collect(toList());
}
protected String name(String name) {
if (name == null) {
return null;
}
return name + "_" + getSanitizedMethodName();
}
}