Add option to load project cache at server startup

In installations with large numbers of projects, it may take a long
time to populate the cache. Allow the option of concurrently loading
the project cache during server startup.

Test result:

- 5700 projects
- Using 40 cpus

Without this change, loading the cache takes about 2 minutes.
With this change, loading the cache takes a few seconds.

Change-Id: Ic05cc8b88ee4c4bf6f5028d3f4dec1e32872aef5

Documentation/config-gerrit.txt

@@ -755,6 +755,26 @@ link:cmd-flush-caches.html[gerrit flush-caches].
 +
 Default is 5 minutes.
 
+[[cache.projects.loadOnStartup]]cache.projects.loadOnStartup::
++
+If the project cache should be loaded during server startup.
++
+The cache is loaded concurrently. Admins should ensure that the cache
+size set under <<cache.name.memoryLimit,cache.projects.memoryLimit>>
+is not smaller than the number of repos.
++
+Default is false, disabled.
+
+[[cache.projects.loadThreads]]cache.projects.loadThreads::
++
+Only relevant if <<cache.projects.loadOnStartup,cache.projects.loadOnStartup>>
+is true.
++
+The number of threads to allocate for loading the cache at startup. These
+threads will die out after the cache is loaded.
++
+Default is the number of CPUs.
+
 [[change]]
 === Section change
 

gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCacheImpl.java

@@ -18,9 +18,11 @@ import com.google.common.base.Throwables;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.Sets;
+import com.google.gerrit.extensions.events.LifecycleListener;
 import com.google.gerrit.reviewdb.client.AccountGroup;
 import com.google.gerrit.reviewdb.client.Project;
 import com.google.gerrit.server.cache.CacheModule;
+import com.google.gerrit.server.project.ProjectCacheWarmer;
 import com.google.gerrit.server.config.AllProjectsName;
 import com.google.gerrit.server.config.AllUsersName;
 import com.google.gerrit.server.git.GitRepositoryManager;
@@ -29,6 +31,7 @@ import com.google.inject.Inject;
 import com.google.inject.Module;
 import com.google.inject.Singleton;
 import com.google.inject.TypeLiteral;
+import com.google.inject.internal.UniqueAnnotations;
 import com.google.inject.name.Named;
 
 import org.eclipse.jgit.errors.RepositoryNotFoundException;
@@ -70,6 +73,9 @@ public class ProjectCacheImpl implements ProjectCache {
 
         bind(ProjectCacheImpl.class);
         bind(ProjectCache.class).to(ProjectCacheImpl.class);
+        bind(LifecycleListener.class)
+          .annotatedWith(UniqueAnnotations.create())
+          .to(ProjectCacheWarmer.class);
       }
     };
   }

gerrit-server/src/main/java/com/google/gerrit/server/project/ProjectCacheWarmer.java

@@ -0,0 +1,75 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.project;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import org.eclipse.jgit.lib.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+
+@Singleton
+public class ProjectCacheWarmer implements LifecycleListener {
+  private static final Logger log =
+      LoggerFactory.getLogger(ProjectCacheWarmer.class);
+
+  private final Config config;
+  private final ProjectCache cache;
+
+  @Inject
+  ProjectCacheWarmer(@GerritServerConfig Config config, ProjectCache cache) {
+    this.config = config;
+    this.cache = cache;
+  }
+
+  @Override
+  public void start() {
+    int cpus = Runtime.getRuntime().availableProcessors();
+    if (config.getBoolean("cache", "projects", "loadOnStartup", false)) {
+      final ThreadPoolExecutor pool =
+          new ScheduledThreadPoolExecutor(config.getInt("cache", "projects",
+              "loadThreads", cpus), new ThreadFactoryBuilder().setNameFormat(
+              "ProjectCacheLoader-%d").build());
+
+      log.info("Loading project cache");
+      pool.execute(new Runnable() {
+        @Override
+        public void run() {
+          for (final Project.NameKey name : cache.all()) {
+            pool.execute(new Runnable() {
+              @Override
+              public void run() {
+                cache.get(name);
+              }
+            });
+          }
+          pool.shutdown();
+        }
+      });
+    }
+  }
+
+  @Override
+  public void stop() {
+  }
+}