show-caches: Improve memory reporting

The way memory was reported was almost useless. Change it to show
the actual values, and the equation that determines how these are
put together to form the current usage.

Include some additional data including server version, current time,
process uptime, active SSH connections, and tasks in the task queue.

The --show-jvm option will report additional data about the JVM,
and tell the caller where it is running.

Change-Id: I80af3cc1d00ef13c985ade3d5faaab792554bac6

Documentation/cmd-show-caches.txt

@@ -8,12 +8,23 @@ gerrit show-caches - Display current cache statistics
 SYNOPSIS
 --------
 [verse]
-'ssh' -p <port> <host> 'gerrit show-caches'
+'ssh' -p <port> <host> 'gerrit show-caches' [--gc] [--show-jvm]
 
 DESCRIPTION
 -----------
 Display statistics about the size and hit ratio of in-memory caches.
 
+OPTIONS
+-------
+--gc::
+	Request Java garbage collection before displaying information
+	about the Java memory heap.
+
+--show-jvm::
+	List the name and version of the Java virtual machine, host
+	operating system, and other details about the environment
+	that Gerrit Code Review is running in.
+
 ACCESS
 ------
 Caller must be a member of the privileged 'Administrators' group,
@@ -28,27 +39,33 @@ EXAMPLES
 
 ====
 	$ ssh -p 29418 review.example.com gerrit show-caches
+	Gerrit Code Review        2.2.2                     now    10:03:34   PDT
+	                                                 uptime     1 min 39 sec
+	
 	  Name               Max |Object Count        |  AvgGet  |Hit Ratio     |
 	                     Age |  Disk    Mem    Cnt|          |Disk Mem  Agg |
 	-------------------------+--------------------+----------+--------------+
-	  accounts           90d |                 295|          |           99%|
+	  accounts           90d |                   1|          |           95%|
-	  accounts_byemail   90d |                 109|          |           97%|
+	  accounts_byemail   90d |                    |          |              |
-	D diff               90d |  2695    128   2707|   0.4ms  | 11%  86%  98%|
+	  accounts_byname    90d |                   1|          |              |
-	  groups             90d |                  94|          |           80%|
+	  adv_bases          10m |                    |          |              |
-	  openid             5m  |                  30|   0.4ms  |            9%|
+	D diff               90d |     8             8|          |              |
-	  projects           90d |                 188|          |           99%|
+	D diff_intraline     90d |     1             1|          |              |
-	  sshkeys            90d |                   9|          |           94%|
+	  groups             90d |                  19|          |            0%|
-	D web_sessions       12h |           30     30|          |  0%  99%  99%|
+	  groups_byext       90d |                    |          |              |
+	  groups_byinclude   90d |                  21|          |           80%|
+	  groups_byname      90d |                    |          |              |
+	  groups_byuuid      90d |                    |          |              |
+	  project_list       90d |                    |          |              |
+	  projects           90d |                   1|          |           80%|
+	  sshkeys            90d |                   1|          |           90%|
+	D web_sessions       12h |                    |          |              |
 	
-	JGit Buffer Cache:
+	SSH:      1  users, oldest session started 782 ms ago
-	  open files  :              23
+	Tasks:    2  total =    1 running +      0 ready +    1 sleeping
-	  loaded      :   6.82 mb
+	Mem:  46.13m total =  16.17m used +  29.96m free +   0.00k buffers
-	  mem%        :   2%
+	     246.56m max
-
+	           0 open files,        6 cpus available,       23 threads
-	JVM Heap:
-	  max         : 880.00 mb
-	  inuse       : 136.57 mb
-	  mem%        :  44%
 ====
 
 SEE ALSO

gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java

@@ -14,6 +14,7 @@
 
 package com.google.gerrit.sshd.commands;
 
+import com.google.gerrit.lifecycle.LifecycleModule;
 import com.google.gerrit.sshd.CommandModule;
 import com.google.gerrit.sshd.CommandName;
 import com.google.gerrit.sshd.Commands;
@@ -58,5 +59,12 @@ public class DefaultCommandModule extends CommandModule {
     command("gerrit-receive-pack").to(Commands.key(git, "receive-pack"));
 
     command("suexec").to(SuExec.class);
+
+    install(new LifecycleModule() {
+      @Override
+      protected void configure() {
+        listener().to(ShowCaches.StartupListener.class);
+      }
+    });
   }
 }

gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/ShowCaches.java

@@ -14,23 +14,71 @@
 
 package com.google.gerrit.sshd.commands;
 
+import com.google.gerrit.common.Version;
+import com.google.gerrit.lifecycle.LifecycleListener;
 import com.google.gerrit.server.IdentifiedUser;
+import com.google.gerrit.server.config.SitePath;
+import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.server.git.WorkQueue.Task;
 import com.google.gerrit.sshd.BaseCommand;
+import com.google.gerrit.sshd.SshDaemon;
 import com.google.inject.Inject;
 
 import net.sf.ehcache.Ehcache;
 import net.sf.ehcache.Statistics;
 import net.sf.ehcache.config.CacheConfiguration;
 
+import org.apache.mina.core.service.IoAcceptor;
+import org.apache.mina.core.session.IoSession;
 import org.apache.sshd.server.Environment;
 import org.eclipse.jgit.storage.file.WindowCacheStatAccessor;
+import org.kohsuke.args4j.Option;
 
+import java.io.File;
+import java.io.IOException;
 import java.io.PrintWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Date;
 
 /** Show the current cache states. */
 final class ShowCaches extends CacheCommand {
+  private static volatile long serverStarted;
+
+  static class StartupListener implements LifecycleListener {
+    @Override
+    public void start() {
+      serverStarted = System.currentTimeMillis();
+    }
+
+    @Override
+    public void stop() {
+    }
+  }
+
+  @Option(name = "--gc", usage = "perform Java GC before printing memory stats")
+  private boolean gc;
+
+  @Option(name = "--show-jvm", usage = "show details about the JVM")
+  private boolean showJVM;
+
   @Inject
-  IdentifiedUser currentUser;
+  private IdentifiedUser currentUser;
+
+  @Inject
+  private WorkQueue workQueue;
+
+  @Inject
+  private SshDaemon daemon;
+
+  @Inject
+  @SitePath
+  private File sitePath;
 
   private PrintWriter p;
 
@@ -55,6 +103,18 @@ final class ShowCaches extends CacheCommand {
   private void display() {
     p = toPrintWriter(out);
 
+    Date now = new Date();
+    p.format(
+        "%-25s %-20s      now  %16s\n",
+        "Gerrit Code Review",
+        Version.getVersion() != null ? Version.getVersion() : "",
+        new SimpleDateFormat("HH:mm:ss   zzz").format(now));
+    p.format(
+        "%-25s %-20s   uptime %16s\n",
+        "", "",
+        uptime(now.getTime() - serverStarted));
+    p.print('\n');
+
     p.print(String.format(//
         "%1s %-18s %-4s|%-20s|  %-5s  |%-14s|\n" //
         , "" //
@@ -77,8 +137,8 @@ final class ShowCaches extends CacheCommand {
         , "Mem" //
         , "Agg" //
     ));
-    p.println("------------------"
+    p.print("------------------"
-        + "-------+--------------------+----------+--------------+");
+        + "-------+--------------------+----------+--------------+\n");
     for (final Ehcache cache : getAllCaches()) {
       final CacheConfiguration cfg = cache.getCacheConfiguration();
       final boolean useDisk = cfg.isDiskPersistent() || cfg.isOverflowToDisk();
@@ -111,49 +171,152 @@ final class ShowCaches extends CacheCommand {
             ));
       }
     }
-    p.println();
+    p.print('\n');
 
+    if (gc) {
+      System.gc();
+      System.runFinalization();
+      System.gc();
+    }
+
+    sshSummary();
+    taskSummary();
+    memSummary();
+
+    if (showJVM) {
+      jvmSummary();
+    }
+
+    p.flush();
+  }
+
+  private void memSummary() {
     final Runtime r = Runtime.getRuntime();
     final long mMax = r.maxMemory();
     final long mFree = r.freeMemory();
     final long mTotal = r.totalMemory();
     final long mInuse = mTotal - mFree;
+
+    final int jgitOpen = WindowCacheStatAccessor.getOpenFiles();
     final long jgitBytes = WindowCacheStatAccessor.getOpenBytes();
 
-    p.println("JGit Buffer Cache:");
+    p.format("Mem: %s total = %s used + %s free + %s buffers\n",
-    fItemCount("open files", WindowCacheStatAccessor.getOpenFiles());
+        bytes(mTotal),
-    fByteCount("loaded", jgitBytes);
+        bytes(mInuse - jgitBytes),
-    fPercent("mem%", jgitBytes, mTotal);
+        bytes(mFree),
-    p.println();
+        bytes(jgitBytes));
-
+    p.format("     %s max\n", bytes(mMax));
-    p.println("JVM Heap:");
+    p.format("    %8d open files, %8d cpus available, %8d threads\n",
-    fByteCount("max", mMax);
+        jgitOpen,
-    fByteCount("inuse", mInuse);
+        r.availableProcessors(),
-    fPercent("mem%", mInuse, mTotal);
+        ManagementFactory.getThreadMXBean().getThreadCount());
-    p.println();
+    p.print('\n');
-
-    p.flush();
   }
 
-  private void fItemCount(final String name, final long value) {
+  private void taskSummary() {
-    p.println(String.format("  %1$-12s: %2$15d", name, value));
+    Collection<Task<?>> pending = workQueue.getTasks();
+    int tasksTotal = pending.size();
+    int tasksRunning = 0, tasksReady = 0, tasksSleeping = 0;
+    for (Task<?> task : pending) {
+      switch (task.getState()) {
+        case RUNNING: tasksRunning++; break;
+        case READY: tasksReady++; break;
+        case SLEEPING: tasksSleeping++; break;
+      }
+    }
+    p.format(
+        "Tasks: %4d  total = %4d running +   %4d ready + %4d sleeping\n",
+        tasksTotal,
+        tasksRunning,
+        tasksReady,
+        tasksSleeping);
   }
 
-  private void fByteCount(final String name, double value) {
+  private void sshSummary() {
-    String suffix = "bytes";
+    IoAcceptor acceptor = daemon.getIoAcceptor();
+    if (acceptor == null) {
+      return;
+    }
+
+    long now = System.currentTimeMillis();
+    Collection<IoSession> list = acceptor.getManagedSessions().values();
+    long oldest = now;
+    double writeThroughput = 0.0;
+    for (IoSession s : list) {
+      oldest = Math.min(oldest, s.getCreationTime());
+    }
+
+    p.format(
+        "SSH:   %4d  users, oldest session started %s ago\n",
+        list.size(),
+        uptime(now - oldest));
+  }
+
+  private void jvmSummary() {
+    OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
+    RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
+    p.format("JVM: %s %s %s\n",
+        runtimeBean.getVmVendor(),
+        runtimeBean.getVmName(),
+        runtimeBean.getVmVersion());
+    p.format("  on %s %s %s\n", "",
+        osBean.getName(),
+        osBean.getVersion(),
+        osBean.getArch());
+    try {
+      p.format("  running as %s on %s\n",
+          System.getProperty("user.name"),
+          InetAddress.getLocalHost().getHostName());
+    } catch (UnknownHostException e) {
+    }
+    p.format("  cwd  %s\n", path(new File(".").getAbsoluteFile().getParentFile()));
+    p.format("  site %s\n", path(sitePath));
+  }
+
+  private String path(File file) {
+    try {
+      return file.getCanonicalPath();
+    } catch (IOException err) {
+      return file.getAbsolutePath();
+    }
+  }
+
+  private String uptime(long uptimeMillis) {
+    if (uptimeMillis < 1000) {
+      return String.format("%3d ms", uptimeMillis);
+    }
+
+    long uptime = uptimeMillis / 1000L;
+
+    long min = uptime / 60;
+    if (min < 60) {
+      return String.format("%2d min %2d sec", min, uptime - min * 60);
+    }
+
+    long hr = uptime / 3600;
+    if (hr < 24) {
+      min = (uptime - hr * 3600) / 60;
+      return String.format("%2d hrs %2d min", hr, min);
+    }
+
+    long days = uptime / (24 * 3600);
+    hr = (uptime - (days * 24 * 3600)) / 3600;
+    return String.format("%4d days %2d hrs", days, hr);
+  }
+
+  private String bytes(double value) {
+    value /= 1024;
+    String suffix = "k";
+
     if (value > 1024) {
       value /= 1024;
-      suffix = "kb";
+      suffix = "m";
     }
     if (value > 1024) {
       value /= 1024;
-      suffix = "mb";
+      suffix = "g";
     }
-    if (value > 1024) {
+    return String.format("%1$6.2f%2$s", value, suffix);
-      value /= 1024;
-      suffix = "gb";
-    }
-    p.println(String.format("  %1$-12s: %2$6.2f %3$s", name, value, suffix));
   }
 
   private String count(long cnt) {
@@ -211,9 +374,4 @@ final class ShowCaches extends CacheCommand {
     final long pcent = (100 * value) / total;
     return String.format("%3d%%", (int) pcent);
   }
-
-  private void fPercent(final String name, final long value, final long total) {
-    final long pcent = 0 < total ? (100 * value) / total : 0;
-    p.println(String.format("  %1$-12s: %2$3d%%", name, (int) pcent));
-  }
 }