opendev/gerrit
Code Issues Proposed changes

Back in-memory caches with Guava, disk caches with H2

Browse Source 
Instead of using Ehcache for in-memory caches, use Guava. The Guava
cache code has been more completely tested by Google in high load
production environments, and it tends to have fewer bugs. It enables
caches to be built at any time, rather than only at server startup.

By creating a Guava cache as soon as it is declared, rather than
during the LifecycleListener.start() for the CachePool, we can promise
any downstream consumer of the cache that the cache is ready to
execute requests the moment it is supplied by Guice. This fixes a
startup ordering problem in the GroupCache and the ProjectCache, where
code wants to use one of these caches during startup to resolve a
group or project by name.

Tracking the Gauva backend caches with a DynamicMap makes it possible
for plugins to define their own in-memory caches using CacheModule's
cache() function to declare the cache. It allows the core server to
make the cache available to administrators over SSH with the gerrit
show-caches and gerrit flush-caches commands.

Persistent caches store in a private H2 database per cache, with a
simple one-table schema that stores each entry in a table row as a
pair of serialized objects (key and value). Database reads are gated
by a BloomFilter, to reduce the number of calls made to H2 during
cache misses. In theory less than 3% of cache misses will reach H2 and
find nothing. Stores happen on a background thread quickly after the
put is made to the cache, reducing the risk that a diff or web_session
record is lost during an ungraceful shutdown.

Cache databases are capped around 128M worth of stored data by running
a prune cycle each day at 1 AM local server time. Records are removed
from the database by ordering on the last access time, where last
accessed is the last time the record was moved from disk to memory.

Change-Id: Ia82d056796b5af9bcb1f219fe06d905c9c0fbc84
This commit is contained in:
Shawn O. Pearce
2012-05-24 14:28:40 -07:00
parent 34d4d1929a
commit 2e1cb2b849
76 changed files with 2432 additions and 1674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

192
gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/ShowCaches.java
View File

@@ -14,22 +14,24 @@
package com.google.gerrit.sshd.commands;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheStats;
import com.google.common.collect.Maps;
import com.google.gerrit.common.Version;
import com.google.gerrit.common.data.GlobalCapability;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.server.cache.h2.H2CacheImpl;
import com.google.gerrit.server.config.SitePath;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.git.WorkQueue.Task;
import com.google.gerrit.sshd.RequiresCapability;
import com.google.gerrit.sshd.SshDaemon;
import com.google.inject.Inject;
import net.sf.ehcache.Ehcache;
import net.sf.ehcache.Statistics;
import net.sf.ehcache.config.CacheConfiguration;
import com.google.inject.Provider;
import org.apache.mina.core.service.IoAcceptor;
import org.apache.mina.core.session.IoSession;
import org.apache.sshd.server.Environment;
import org.eclipse.jgit.storage.file.WindowCacheStatAccessor;
import org.kohsuke.args4j.Option;
@@ -43,6 +45,8 @@ import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.Map;
import java.util.SortedMap;
/** Show the current cache states. */
@RequiresCapability(GlobalCapability.VIEW_CACHES)
@@ -76,8 +80,26 @@ final class ShowCaches extends CacheCommand {
@SitePath
private File sitePath;
@Option(name = "--width", aliases = {"-w"}, metaVar = "COLS", usage = "width of output table")
private int columns = 80;
private int nw;
@Override
public void start(Environment env) throws IOException {
String s = env.getEnv().get(Environment.ENV_COLUMNS);
if (s != null && !s.isEmpty()) {
try {
columns = Integer.parseInt(s);
} catch (NumberFormatException err) {
columns = 80;
}
}
super.start(env);
}
@Override
protected void run() {
nw = columns - 50;
Date now = new Date();
stdout.format(
"%-25s %-20s now %16s\n",
@@ -91,60 +113,46 @@ final class ShowCaches extends CacheCommand {
stdout.print('\n');
stdout.print(String.format(//
"%1s %-18s %-4s|%-20s| %-5s |%-14s|\n" //
"%1s %-"+nw+"s|%-21s| %-5s |%-9s|\n" //
, "" //
, "Name" //
, "Max" //
, "Object Count" //
, "Entries" //
, "AvgGet" //
, "Hit Ratio" //
));
stdout.print(String.format(//
"%1s %-18s %-4s|%6s %6s %6s| %-5s |%-4s %-4s %-4s|\n" //
"%1s %-"+nw+"s|%6s %6s %7s| %-5s |%-4s %-4s|\n" //
, "" //
, "" //
, "Age" //
, "Disk" //
, "Mem" //
, "Cnt" //
, "" //
, "Disk" //
, "Space" //
, "" //
, "Mem" //
, "Agg" //
, "Disk" //
));
stdout.print("------------------"
+ "-------+--------------------+----------+--------------+\n");
for (final Ehcache cache : getAllCaches()) {
final CacheConfiguration cfg = cache.getCacheConfiguration();
final boolean useDisk = cfg.isDiskPersistent() || cfg.isOverflowToDisk();
final Statistics stat = cache.getStatistics();
final long total = stat.getCacheHits() + stat.getCacheMisses();
stdout.print("--");
for (int i = 0; i < nw; i++) {
stdout.print('-');
}
stdout.print("+---------------------+---------+---------+\n");
if (useDisk) {
stdout.print(String.format(//
"D %-18s %-4s|%6s %6s %6s| %7s |%4s %4s %4s|\n" //
, cache.getName() //
, interval(cfg.getTimeToLiveSeconds()) //
, count(stat.getDiskStoreObjectCount()) //
, count(stat.getMemoryStoreObjectCount()) //
, count(stat.getObjectCount()) //
, duration(stat.getAverageGetTime()) //
, percent(stat.getOnDiskHits(), total) //
, percent(stat.getInMemoryHits(), total) //
, percent(stat.getCacheHits(), total) //
));
} else {
stdout.print(String.format(//
" %-18s %-4s|%6s %6s %6s| %7s |%4s %4s %4s|\n" //
, cache.getName() //
, interval(cfg.getTimeToLiveSeconds()) //
, "", "" //
, count(stat.getObjectCount()) //
, duration(stat.getAverageGetTime()) //
, "", "" //
, percent(stat.getCacheHits(), total) //
));
}
Map<String, H2CacheImpl<?, ?>> disks = Maps.newTreeMap();
printMemoryCaches(disks, sortedCoreCaches());
printMemoryCaches(disks, sortedPluginCaches());
for (Map.Entry<String, H2CacheImpl<?, ?>> entry : disks.entrySet()) {
H2CacheImpl<?, ?> cache = entry.getValue();
CacheStats stat = cache.stats();
H2CacheImpl.DiskStats disk = cache.diskStats();
stdout.print(String.format(
"D %-"+nw+"s|%6s %6s %7s| %7s |%4s %4s|\n",
entry.getKey(),
count(cache.size()),
count(disk.size()),
bytes(disk.space()),
duration(stat.averageLoadPenalty()),
percent(stat.hitCount(), stat.requestCount()),
percent(disk.hitCount(), disk.requestCount())));
}
stdout.print('\n');
@@ -165,6 +173,51 @@ final class ShowCaches extends CacheCommand {
stdout.flush();
}
private void printMemoryCaches(
Map<String, H2CacheImpl<?, ?>> disks,
Map<String, Cache<?,?>> caches) {
for (Map.Entry<String, Cache<?,?>> entry : caches.entrySet()) {
Cache<?,?> cache = entry.getValue();
if (cache instanceof H2CacheImpl) {
disks.put(entry.getKey(), (H2CacheImpl<?,?>)cache);
continue;
}
CacheStats stat = cache.stats();
stdout.print(String.format(
" %-"+nw+"s|%6s %6s %7s| %7s |%4s %4s|\n",
entry.getKey(),
count(cache.size()),
"",
"",
duration(stat.averageLoadPenalty()),
percent(stat.hitCount(), stat.requestCount()),
""));
}
}
private Map<String, Cache<?, ?>> sortedCoreCaches() {
SortedMap<String, Cache<?, ?>> m = Maps.newTreeMap();
for (Map.Entry<String, Provider<Cache<?, ?>>> entry :
cacheMap.byPlugin("gerrit").entrySet()) {
m.put(cacheNameOf("gerrit", entry.getKey()), entry.getValue().get());
}
return m;
}
private Map<String, Cache<?, ?>> sortedPluginCaches() {
SortedMap<String, Cache<?, ?>> m = Maps.newTreeMap();
for (String plugin : cacheMap.plugins()) {
if ("gerrit".equals(plugin)) {
continue;
}
for (Map.Entry<String, Provider<Cache<?, ?>>> entry :
cacheMap.byPlugin(plugin).entrySet()) {
m.put(cacheNameOf(plugin, entry.getKey()), entry.getValue().get());
}
}
return m;
}
private void memSummary() {
final Runtime r = Runtime.getRuntime();
final long mMax = r.maxMemory();
@@ -300,45 +353,24 @@ final class ShowCaches extends CacheCommand {
return String.format("%6d", cnt);
}
private String duration(double ms) {
if (Math.abs(ms) <= 0.05) {
private String duration(double ns) {
if (ns < 0.5) {
return "";
}
String suffix = "ms";
if (ms >= 1000) {
ms /= 1000;
String suffix = "ns";
if (ns >= 1000.0) {
ns /= 1000.0;
suffix = "us";
}
if (ns >= 1000.0) {
ns /= 1000.0;
suffix = "ms";
}
if (ns >= 1000.0) {
ns /= 1000.0;
suffix = "s ";
}
return String.format("%4.1f%s", ms, suffix);
}
private String interval(double ttl) {
if (ttl == 0) {
return "inf";
}
String suffix = "s";
if (ttl >= 60) {
ttl /= 60;
suffix = "m";
if (ttl >= 60) {
ttl /= 60;
suffix = "h";
}
if (ttl >= 24) {
ttl /= 24;
suffix = "d";
if (ttl >= 365) {
ttl /= 365;
suffix = "y";
}
}
}
return Integer.toString((int) ttl) + suffix;
return String.format("%4.1f%s", ns, suffix);
}
private String percent(final long value, final long total) {