Add option to load project cache at server startup
In installations with large numbers of projects, it may take a long time to populate the cache. Allow the option of concurrently loading the project cache during server startup. Test result: - 5700 projects - Using 40 cpus Without this change, loading the cache takes about 2 minutes. With this change, loading the cache takes a few seconds. Change-Id: Ic05cc8b88ee4c4bf6f5028d3f4dec1e32872aef5
This commit is contained in:
parent
d06ba15d46
commit
fc19ff1899
@ -755,6 +755,26 @@ link:cmd-flush-caches.html[gerrit flush-caches].
|
||||
+
|
||||
Default is 5 minutes.
|
||||
|
||||
[[cache.projects.loadOnStartup]]cache.projects.loadOnStartup::
|
||||
+
|
||||
If the project cache should be loaded during server startup.
|
||||
+
|
||||
The cache is loaded concurrently. Admins should ensure that the cache
|
||||
size set under <<cache.name.memoryLimit,cache.projects.memoryLimit>>
|
||||
is not smaller than the number of repos.
|
||||
+
|
||||
Default is false, disabled.
|
||||
|
||||
[[cache.projects.loadThreads]]cache.projects.loadThreads::
|
||||
+
|
||||
Only relevant if <<cache.projects.loadOnStartup,cache.projects.loadOnStartup>>
|
||||
is true.
|
||||
+
|
||||
The number of threads to allocate for loading the cache at startup. These
|
||||
threads will die out after the cache is loaded.
|
||||
+
|
||||
Default is the number of CPUs.
|
||||
|
||||
[[change]]
|
||||
=== Section change
|
||||
|
||||
|
@ -18,9 +18,11 @@ import com.google.common.base.Throwables;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.gerrit.extensions.events.LifecycleListener;
|
||||
import com.google.gerrit.reviewdb.client.AccountGroup;
|
||||
import com.google.gerrit.reviewdb.client.Project;
|
||||
import com.google.gerrit.server.cache.CacheModule;
|
||||
import com.google.gerrit.server.project.ProjectCacheWarmer;
|
||||
import com.google.gerrit.server.config.AllProjectsName;
|
||||
import com.google.gerrit.server.config.AllUsersName;
|
||||
import com.google.gerrit.server.git.GitRepositoryManager;
|
||||
@ -29,6 +31,7 @@ import com.google.inject.Inject;
|
||||
import com.google.inject.Module;
|
||||
import com.google.inject.Singleton;
|
||||
import com.google.inject.TypeLiteral;
|
||||
import com.google.inject.internal.UniqueAnnotations;
|
||||
import com.google.inject.name.Named;
|
||||
|
||||
import org.eclipse.jgit.errors.RepositoryNotFoundException;
|
||||
@ -70,6 +73,9 @@ public class ProjectCacheImpl implements ProjectCache {
|
||||
|
||||
bind(ProjectCacheImpl.class);
|
||||
bind(ProjectCache.class).to(ProjectCacheImpl.class);
|
||||
bind(LifecycleListener.class)
|
||||
.annotatedWith(UniqueAnnotations.create())
|
||||
.to(ProjectCacheWarmer.class);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
@ -0,0 +1,75 @@
|
||||
// Copyright (C) 2014 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.server.project;
|
||||
|
||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
import com.google.gerrit.extensions.events.LifecycleListener;
|
||||
import com.google.gerrit.reviewdb.client.Project;
|
||||
import com.google.gerrit.server.config.GerritServerConfig;
|
||||
import com.google.inject.Inject;
|
||||
import com.google.inject.Singleton;
|
||||
|
||||
import org.eclipse.jgit.lib.Config;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.concurrent.ScheduledThreadPoolExecutor;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
|
||||
@Singleton
|
||||
public class ProjectCacheWarmer implements LifecycleListener {
|
||||
private static final Logger log =
|
||||
LoggerFactory.getLogger(ProjectCacheWarmer.class);
|
||||
|
||||
private final Config config;
|
||||
private final ProjectCache cache;
|
||||
|
||||
@Inject
|
||||
ProjectCacheWarmer(@GerritServerConfig Config config, ProjectCache cache) {
|
||||
this.config = config;
|
||||
this.cache = cache;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
int cpus = Runtime.getRuntime().availableProcessors();
|
||||
if (config.getBoolean("cache", "projects", "loadOnStartup", false)) {
|
||||
final ThreadPoolExecutor pool =
|
||||
new ScheduledThreadPoolExecutor(config.getInt("cache", "projects",
|
||||
"loadThreads", cpus), new ThreadFactoryBuilder().setNameFormat(
|
||||
"ProjectCacheLoader-%d").build());
|
||||
|
||||
log.info("Loading project cache");
|
||||
pool.execute(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
for (final Project.NameKey name : cache.all()) {
|
||||
pool.execute(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
cache.get(name);
|
||||
}
|
||||
});
|
||||
}
|
||||
pool.shutdown();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user