Revert "Temporarily disable reindexIfStale for accounts"

The account staleness checker was disabled because of failures in the
Elasticsearch tests that looked like [1]. The investigation of this
issue showed that this was not caused by the account staleness checker
but was an issue that could already occur before. For some reason the
issue is hit more often when the staleness checker is enabled. Change
I7b20a0cee6 fixed this partly, but the problem can still occur. In
general the unstable Elasticsearch tests should not prevent us from
enabling unrelated features like the staleness checker for accounts.
The CI already excludes the Elasticsearch tests since they are flaky.
Fixing the flakyness of the Elasticsearch tests is unrelated to the
account staleness checker and should be done in separate changes.
However to not make the Elasticsearch tests any more flaky disable the
staleness checker for all Elasticsearch tests.

However a real issue with enabling the account staleness checker was
that it made some account tests flaky. Some account tests verify the
number of times that an account is indexed. With the account staleness
checker these tests got flaky because the staleness checker caused extra
reindex events. This may happen if an account is reindexed several times
in a row since the staleness check is done asynchronously, e.g. if an
account is reindexed twice the following happens:

1. TestThread: update account to state A, index and trigger reindex if
   stale
2. BatchExecutorThread: auto reindex if stale, triggered by 1.
3. TestThread: update account to state B, index and trigger reindex if
   stale
4. BatchExecutorThread: auto reindex if stale, triggered by 3.

Since 2 and 3 are executed in parallel it can happen that 2 reads state
B from NoteDb and state A from index and hence detects the account as
stale which triggers another reindex. This is fine, it's an extra
reindex event but the logic ensures that in the end the account is never
stale in the index.

However since this is confusing the account tests the staleness checker
is now explicitly disabled for all account tests.

There was also some flakiness observed when we expected only a single
index event [2] but I wasn't able to reproduce this. Disabling the
staleness checker for all account tests makes sure that we don't get
this flakiness either.

This reverts commit b316186826.

[1]
[accounts_0007] IndexAlreadyExistsException[already exists]
  at org.elasticsearch.cluster.metadata.MetaDataCreateIndexService.validateIndexName(MetaDataCreateIndexService.java:136)
  at org.elasticsearch.cluster.metadata.MetaDataCreateIndexService.validate(MetaDataCreateIndexService.java:431)
  at org.elasticsearch.cluster.metadata.MetaDataCreateIndexService.access$100(MetaDataCreateIndexService.java:95)
  at org.elasticsearch.cluster.metadata.MetaDataCreateIndexService$1.execute(MetaDataCreateIndexService.java:190)
  at org.elasticsearch.cluster.ClusterStateUpdateTask.execute(ClusterStateUpdateTask.java:45)
  at org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:480)
  at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  at java.lang.Thread.run(Thread.java:745)

[2] https://gerrit-ci.gerritforge.com/job/Gerrit-verifier-bazel/35988/consoleText

Change-Id: Ib0c69dd6805b2624679afa10d2ef2fd89dc5f8be
Signed-off-by: Edwin Kempin <ekempin@google.com>

java/com/google/gerrit/server/index/account/AccountIndexerImpl.java

@@ -102,7 +102,7 @@ public class AccountIndexerImpl implements AccountIndexer {
   }
 
   private static boolean autoReindexIfStale(Config cfg) {
-    return cfg.getBoolean("index", null, "autoReindexIfStale", false);
+    return cfg.getBoolean("index", null, "autoReindexIfStale", true);
   }
 
   private void autoReindexIfStale(Account.Id id) {

javatests/com/google/gerrit/acceptance/api/accounts/AccountIT.java

@@ -150,6 +150,11 @@ public class AccountIT extends AbstractDaemonTest {
   public static Config enableSignedPushConfig() {
     Config cfg = new Config();
     cfg.setBoolean("receive", null, "enableSignedPush", true);
+
+    // Disable the staleness checker so that tests that verify the number of expected index events
+    // are stable.
+    cfg.setBoolean("index", null, "autoReindexIfStale", false);
+
     return cfg;
   }
 

javatests/com/google/gerrit/elasticsearch/BUILD

@@ -40,6 +40,7 @@ ELASTICSEARCH_TESTS = {i: "ElasticQuery" + i.capitalize() + "sTest.java" for i i
         "//java/com/google/gerrit/server",
         "//java/com/google/gerrit/server/project/testing:project-test-util",
         "//java/com/google/gerrit/testing:gerrit-test-util",
+        "//javatests/com/google/gerrit/server/query:index-config",
         "//javatests/com/google/gerrit/server/query/%s:abstract_query_tests" % name,
         "//lib/guice",
         "//lib/jgit/org.eclipse.jgit:jgit",

javatests/com/google/gerrit/elasticsearch/ElasticQueryAccountsTest.java

@@ -15,7 +15,9 @@
 package com.google.gerrit.elasticsearch;
 
 import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
+import com.google.gerrit.server.query.IndexConfig;
 import com.google.gerrit.server.query.account.AbstractQueryAccountsTest;
+import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.InMemoryModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -26,6 +28,11 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 public class ElasticQueryAccountsTest extends AbstractQueryAccountsTest {
+  @ConfigSuite.Default
+  public static Config defaultConfig() {
+    return IndexConfig.createForElasticsearch();
+  }
+
   private static ElasticNodeInfo nodeInfo;
 
   @BeforeClass

javatests/com/google/gerrit/elasticsearch/ElasticQueryChangesTest.java

@@ -15,7 +15,9 @@
 package com.google.gerrit.elasticsearch;
 
 import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
+import com.google.gerrit.server.query.IndexConfig;
 import com.google.gerrit.server.query.change.AbstractQueryChangesTest;
+import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.InMemoryModule;
 import com.google.gerrit.testing.InMemoryRepositoryManager.Repo;
 import com.google.inject.Guice;
@@ -29,6 +31,11 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class ElasticQueryChangesTest extends AbstractQueryChangesTest {
+  @ConfigSuite.Default
+  public static Config defaultConfig() {
+    return IndexConfig.createForElasticsearch();
+  }
+
   private static ElasticNodeInfo nodeInfo;
 
   @BeforeClass

javatests/com/google/gerrit/elasticsearch/ElasticQueryGroupsTest.java

@@ -15,7 +15,9 @@
 package com.google.gerrit.elasticsearch;
 
 import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
+import com.google.gerrit.server.query.IndexConfig;
 import com.google.gerrit.server.query.group.AbstractQueryGroupsTest;
+import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.InMemoryModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -26,6 +28,11 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 public class ElasticQueryGroupsTest extends AbstractQueryGroupsTest {
+  @ConfigSuite.Default
+  public static Config defaultConfig() {
+    return IndexConfig.createForElasticsearch();
+  }
+
   private static ElasticNodeInfo nodeInfo;
 
   @BeforeClass

javatests/com/google/gerrit/elasticsearch/ElasticQueryProjectsTest.java

@@ -15,7 +15,9 @@
 package com.google.gerrit.elasticsearch;
 
 import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
+import com.google.gerrit.server.query.IndexConfig;
 import com.google.gerrit.server.query.project.AbstractQueryProjectsTest;
+import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.InMemoryModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -26,6 +28,11 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 public class ElasticQueryProjectsTest extends AbstractQueryProjectsTest {
+  @ConfigSuite.Default
+  public static Config defaultConfig() {
+    return IndexConfig.createForElasticsearch();
+  }
+
   private static ElasticNodeInfo nodeInfo;
 
   @BeforeClass

javatests/com/google/gerrit/server/query/BUILD

@@ -0,0 +1,10 @@
+load("//tools/bzl:junit.bzl", "junit_tests")
+
+java_library(
+    name = "index-config",
+    srcs = glob(["*.java"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//lib/jgit/org.eclipse.jgit:jgit",
+    ],
+)

javatests/com/google/gerrit/server/query/IndexConfig.java

@@ -0,0 +1,40 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.query;
+
+import org.eclipse.jgit.lib.Config;
+
+public class IndexConfig {
+
+  public static Config createForLucene() {
+    return create();
+  }
+
+  public static Config createForElasticsearch() {
+    Config cfg = create();
+
+    // For some reason enabling the staleness checker increases the flakiness of the Elasticsearch
+    // tests. Hence disable the staleness checker.
+    cfg.setBoolean("index", null, "autoReindexIfStale", false);
+
+    return cfg;
+  }
+
+  public static Config create() {
+    Config cfg = new Config();
+    cfg.setInt("index", null, "maxPages", 10);
+    return cfg;
+  }
+}

javatests/com/google/gerrit/server/query/account/AbstractQueryAccountsTest.java

@@ -60,7 +60,6 @@ import com.google.gerrit.server.util.ManualRequestContext;
 import com.google.gerrit.server.util.OneOffRequestContext;
 import com.google.gerrit.server.util.RequestContext;
 import com.google.gerrit.server.util.ThreadLocalRequestContext;
-import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.GerritServerTests;
 import com.google.gerrit.testing.InMemoryDatabase;
 import com.google.inject.Inject;
@@ -72,7 +71,6 @@ import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Optional;
-import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.lib.PersonIdent;
 import org.eclipse.jgit.lib.Repository;
 import org.junit.After;
@@ -82,13 +80,6 @@ import org.junit.Test;
 
 @Ignore
 public abstract class AbstractQueryAccountsTest extends GerritServerTests {
-  @ConfigSuite.Default
-  public static Config defaultConfig() {
-    Config cfg = new Config();
-    cfg.setInt("index", null, "maxPages", 10);
-    return cfg;
-  }
-
   @Inject protected Accounts accounts;
 
   @Inject protected AccountsUpdate.Server accountsUpdate;

javatests/com/google/gerrit/server/query/account/BUILD

@@ -32,6 +32,7 @@ junit_tests(
         ":abstract_query_tests",
         "//java/com/google/gerrit/server",
         "//java/com/google/gerrit/testing:gerrit-test-util",
+        "//javatests/com/google/gerrit/server/query:index-config",
         "//lib/guice",
         "//lib/jgit/org.eclipse.jgit:jgit",
     ],

javatests/com/google/gerrit/server/query/account/LuceneQueryAccountsTest.java

@@ -15,6 +15,7 @@
 package com.google.gerrit.server.query.account;
 
 import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
+import com.google.gerrit.server.query.IndexConfig;
 import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.InMemoryModule;
 import com.google.gerrit.testing.IndexVersions;
@@ -25,6 +26,11 @@ import java.util.Map;
 import org.eclipse.jgit.lib.Config;
 
 public class LuceneQueryAccountsTest extends AbstractQueryAccountsTest {
+  @ConfigSuite.Default
+  public static Config defaultConfig() {
+    return IndexConfig.createForLucene();
+  }
+
   @ConfigSuite.Configs
   public static Map<String, Config> againstPreviousIndexVersion() {
     // the current schema version is already tested by the inherited default config suite

javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java

@@ -105,7 +105,6 @@ import com.google.gerrit.server.util.ManualRequestContext;
 import com.google.gerrit.server.util.OneOffRequestContext;
 import com.google.gerrit.server.util.RequestContext;
 import com.google.gerrit.server.util.ThreadLocalRequestContext;
-import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.DisabledReviewDb;
 import com.google.gerrit.testing.GerritServerTests;
 import com.google.gerrit.testing.InMemoryDatabase;
@@ -130,7 +129,6 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import org.eclipse.jgit.junit.TestRepository;
-import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectInserter;
 import org.eclipse.jgit.lib.ObjectReader;
@@ -147,13 +145,6 @@ import org.junit.Test;
 
 @Ignore
 public abstract class AbstractQueryChangesTest extends GerritServerTests {
-  @ConfigSuite.Default
-  public static Config defaultConfig() {
-    Config cfg = new Config();
-    cfg.setInt("index", null, "maxPages", 10);
-    return cfg;
-  }
-
   @Inject protected Accounts accounts;
   @Inject protected AccountCache accountCache;
   @Inject protected AccountsUpdate.Server accountsUpdate;

javatests/com/google/gerrit/server/query/change/BUILD

@@ -39,6 +39,7 @@ junit_tests(
         "//java/com/google/gerrit/reviewdb:server",
         "//java/com/google/gerrit/server",
         "//java/com/google/gerrit/testing:gerrit-test-util",
+        "//javatests/com/google/gerrit/server/query:index-config",
         "//lib:gwtorm",
         "//lib:truth",
         "//lib/guice",

javatests/com/google/gerrit/server/query/change/LuceneQueryChangesTest.java

@@ -17,6 +17,7 @@ package com.google.gerrit.server.query.change;
 import com.google.gerrit.extensions.restapi.BadRequestException;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
+import com.google.gerrit.server.query.IndexConfig;
 import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.InMemoryModule;
 import com.google.gerrit.testing.InMemoryRepositoryManager.Repo;
@@ -31,6 +32,11 @@ import org.eclipse.jgit.revwalk.RevCommit;
 import org.junit.Test;
 
 public class LuceneQueryChangesTest extends AbstractQueryChangesTest {
+  @ConfigSuite.Default
+  public static Config defaultConfig() {
+    return IndexConfig.createForLucene();
+  }
+
   @ConfigSuite.Configs
   public static Map<String, Config> againstPreviousIndexVersion() {
     // the current schema version is already tested by the inherited default config suite

javatests/com/google/gerrit/server/query/group/AbstractQueryGroupsTest.java

@@ -56,7 +56,6 @@ import com.google.gerrit.server.util.ManualRequestContext;
 import com.google.gerrit.server.util.OneOffRequestContext;
 import com.google.gerrit.server.util.RequestContext;
 import com.google.gerrit.server.util.ThreadLocalRequestContext;
-import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.GerritServerTests;
 import com.google.gerrit.testing.InMemoryDatabase;
 import com.google.inject.Inject;
@@ -68,7 +67,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
-import org.eclipse.jgit.lib.Config;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -76,13 +74,6 @@ import org.junit.Test;
 
 @Ignore
 public abstract class AbstractQueryGroupsTest extends GerritServerTests {
-  @ConfigSuite.Default
-  public static Config defaultConfig() {
-    Config cfg = new Config();
-    cfg.setInt("index", null, "maxPages", 10);
-    return cfg;
-  }
-
   @Inject protected Accounts accounts;
 
   @Inject protected AccountsUpdate.Server accountsUpdate;

javatests/com/google/gerrit/server/query/group/BUILD

@@ -32,6 +32,7 @@ junit_tests(
         ":abstract_query_tests",
         "//java/com/google/gerrit/server",
         "//java/com/google/gerrit/testing:gerrit-test-util",
+        "//javatests/com/google/gerrit/server/query:index-config",
         "//lib/guice",
         "//lib/jgit/org.eclipse.jgit:jgit",
     ],

javatests/com/google/gerrit/server/query/group/LuceneQueryGroupsTest.java

@@ -15,6 +15,7 @@
 package com.google.gerrit.server.query.group;
 
 import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
+import com.google.gerrit.server.query.IndexConfig;
 import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.InMemoryModule;
 import com.google.gerrit.testing.IndexVersions;
@@ -25,6 +26,11 @@ import java.util.Map;
 import org.eclipse.jgit.lib.Config;
 
 public class LuceneQueryGroupsTest extends AbstractQueryGroupsTest {
+  @ConfigSuite.Default
+  public static Config defaultConfig() {
+    return IndexConfig.createForLucene();
+  }
+
   @ConfigSuite.Configs
   public static Map<String, Config> againstPreviousIndexVersion() {
     // the current schema version is already tested by the inherited default config suite

javatests/com/google/gerrit/server/query/project/AbstractQueryProjectsTest.java

@@ -43,7 +43,6 @@ import com.google.gerrit.server.util.ManualRequestContext;
 import com.google.gerrit.server.util.OneOffRequestContext;
 import com.google.gerrit.server.util.RequestContext;
 import com.google.gerrit.server.util.ThreadLocalRequestContext;
-import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.GerritServerTests;
 import com.google.gerrit.testing.InMemoryDatabase;
 import com.google.inject.Inject;
@@ -54,7 +53,6 @@ import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
-import org.eclipse.jgit.lib.Config;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -62,13 +60,6 @@ import org.junit.Test;
 
 @Ignore
 public abstract class AbstractQueryProjectsTest extends GerritServerTests {
-  @ConfigSuite.Default
-  public static Config defaultConfig() {
-    Config cfg = new Config();
-    cfg.setInt("index", null, "maxPages", 10);
-    return cfg;
-  }
-
   @Inject protected Accounts accounts;
 
   @Inject protected AccountsUpdate.Server accountsUpdate;

javatests/com/google/gerrit/server/query/project/BUILD

@@ -31,6 +31,7 @@ junit_tests(
         ":abstract_query_tests",
         "//java/com/google/gerrit/server",
         "//java/com/google/gerrit/testing:gerrit-test-util",
+        "//javatests/com/google/gerrit/server/query:index-config",
         "//lib/guice",
         "//lib/jgit/org.eclipse.jgit:jgit",
     ],

javatests/com/google/gerrit/server/query/project/LuceneQueryProjectsTest.java

@@ -15,6 +15,7 @@
 package com.google.gerrit.server.query.project;
 
 import com.google.gerrit.server.index.project.ProjectSchemaDefinitions;
+import com.google.gerrit.server.query.IndexConfig;
 import com.google.gerrit.testing.ConfigSuite;
 import com.google.gerrit.testing.InMemoryModule;
 import com.google.gerrit.testing.IndexVersions;
@@ -25,6 +26,11 @@ import java.util.Map;
 import org.eclipse.jgit.lib.Config;
 
 public class LuceneQueryProjectsTest extends AbstractQueryProjectsTest {
+  @ConfigSuite.Default
+  public static Config defaultConfig() {
+    return IndexConfig.createForLucene();
+  }
+
   @ConfigSuite.Configs
   public static Map<String, Config> againstPreviousIndexVersion() {
     // the current schema version is already tested by the inherited default config suite