show-caches: Improve memory reporting

The way memory was reported was almost useless. Change it to show
the actual values, and the equation that determines how these are
put together to form the current usage.

Include some additional data including server version, current time,
process uptime, active SSH connections, and tasks in the task queue.

The --show-jvm option will report additional data about the JVM,
and tell the caller where it is running.

Change-Id: I80af3cc1d00ef13c985ade3d5faaab792554bac6

gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java

@@ -14,6 +14,7 @@
 
 package com.google.gerrit.sshd.commands;
 
+import com.google.gerrit.lifecycle.LifecycleModule;
 import com.google.gerrit.sshd.CommandModule;
 import com.google.gerrit.sshd.CommandName;
 import com.google.gerrit.sshd.Commands;
@@ -58,5 +59,12 @@ public class DefaultCommandModule extends CommandModule {
     command("gerrit-receive-pack").to(Commands.key(git, "receive-pack"));
 
     command("suexec").to(SuExec.class);
+
+    install(new LifecycleModule() {
+      @Override
+      protected void configure() {
+        listener().to(ShowCaches.StartupListener.class);
+      }
+    });
   }
 }

gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/ShowCaches.java

@@ -14,23 +14,71 @@
 
 package com.google.gerrit.sshd.commands;
 
+import com.google.gerrit.common.Version;
+import com.google.gerrit.lifecycle.LifecycleListener;
 import com.google.gerrit.server.IdentifiedUser;
+import com.google.gerrit.server.config.SitePath;
+import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.server.git.WorkQueue.Task;
 import com.google.gerrit.sshd.BaseCommand;
+import com.google.gerrit.sshd.SshDaemon;
 import com.google.inject.Inject;
 
 import net.sf.ehcache.Ehcache;
 import net.sf.ehcache.Statistics;
 import net.sf.ehcache.config.CacheConfiguration;
 
+import org.apache.mina.core.service.IoAcceptor;
+import org.apache.mina.core.session.IoSession;
 import org.apache.sshd.server.Environment;
 import org.eclipse.jgit.storage.file.WindowCacheStatAccessor;
+import org.kohsuke.args4j.Option;
 
+import java.io.File;
+import java.io.IOException;
 import java.io.PrintWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Date;
 
 /** Show the current cache states. */
 final class ShowCaches extends CacheCommand {
+  private static volatile long serverStarted;
+
+  static class StartupListener implements LifecycleListener {
+    @Override
+    public void start() {
+      serverStarted = System.currentTimeMillis();
+    }
+
+    @Override
+    public void stop() {
+    }
+  }
+
+  @Option(name = "--gc", usage = "perform Java GC before printing memory stats")
+  private boolean gc;
+
+  @Option(name = "--show-jvm", usage = "show details about the JVM")
+  private boolean showJVM;
+
   @Inject
-  IdentifiedUser currentUser;
+  private IdentifiedUser currentUser;
+
+  @Inject
+  private WorkQueue workQueue;
+
+  @Inject
+  private SshDaemon daemon;
+
+  @Inject
+  @SitePath
+  private File sitePath;
 
   private PrintWriter p;
 
@@ -55,6 +103,18 @@ final class ShowCaches extends CacheCommand {
   private void display() {
     p = toPrintWriter(out);
 
+    Date now = new Date();
+    p.format(
+        "%-25s %-20s      now  %16s\n",
+        "Gerrit Code Review",
+        Version.getVersion() != null ? Version.getVersion() : "",
+        new SimpleDateFormat("HH:mm:ss   zzz").format(now));
+    p.format(
+        "%-25s %-20s   uptime %16s\n",
+        "", "",
+        uptime(now.getTime() - serverStarted));
+    p.print('\n');
+
     p.print(String.format(//
         "%1s %-18s %-4s|%-20s|  %-5s  |%-14s|\n" //
         , "" //
@@ -77,8 +137,8 @@ final class ShowCaches extends CacheCommand {
         , "Mem" //
         , "Agg" //
     ));
-    p.println("------------------"
-        + "-------+--------------------+----------+--------------+");
+    p.print("------------------"
+        + "-------+--------------------+----------+--------------+\n");
     for (final Ehcache cache : getAllCaches()) {
       final CacheConfiguration cfg = cache.getCacheConfiguration();
       final boolean useDisk = cfg.isDiskPersistent() || cfg.isOverflowToDisk();
@@ -111,49 +171,152 @@ final class ShowCaches extends CacheCommand {
             ));
       }
     }
-    p.println();
+    p.print('\n');
 
+    if (gc) {
+      System.gc();
+      System.runFinalization();
+      System.gc();
+    }
+
+    sshSummary();
+    taskSummary();
+    memSummary();
+
+    if (showJVM) {
+      jvmSummary();
+    }
+
+    p.flush();
+  }
+
+  private void memSummary() {
     final Runtime r = Runtime.getRuntime();
     final long mMax = r.maxMemory();
     final long mFree = r.freeMemory();
     final long mTotal = r.totalMemory();
     final long mInuse = mTotal - mFree;
+
+    final int jgitOpen = WindowCacheStatAccessor.getOpenFiles();
     final long jgitBytes = WindowCacheStatAccessor.getOpenBytes();
 
-    p.println("JGit Buffer Cache:");
-    fItemCount("open files", WindowCacheStatAccessor.getOpenFiles());
-    fByteCount("loaded", jgitBytes);
-    fPercent("mem%", jgitBytes, mTotal);
-    p.println();
-
-    p.println("JVM Heap:");
-    fByteCount("max", mMax);
-    fByteCount("inuse", mInuse);
-    fPercent("mem%", mInuse, mTotal);
-    p.println();
-
-    p.flush();
+    p.format("Mem: %s total = %s used + %s free + %s buffers\n",
+        bytes(mTotal),
+        bytes(mInuse - jgitBytes),
+        bytes(mFree),
+        bytes(jgitBytes));
+    p.format("     %s max\n", bytes(mMax));
+    p.format("    %8d open files, %8d cpus available, %8d threads\n",
+        jgitOpen,
+        r.availableProcessors(),
+        ManagementFactory.getThreadMXBean().getThreadCount());
+    p.print('\n');
   }
 
-  private void fItemCount(final String name, final long value) {
-    p.println(String.format("  %1$-12s: %2$15d", name, value));
+  private void taskSummary() {
+    Collection<Task<?>> pending = workQueue.getTasks();
+    int tasksTotal = pending.size();
+    int tasksRunning = 0, tasksReady = 0, tasksSleeping = 0;
+    for (Task<?> task : pending) {
+      switch (task.getState()) {
+        case RUNNING: tasksRunning++; break;
+        case READY: tasksReady++; break;
+        case SLEEPING: tasksSleeping++; break;
+      }
+    }
+    p.format(
+        "Tasks: %4d  total = %4d running +   %4d ready + %4d sleeping\n",
+        tasksTotal,
+        tasksRunning,
+        tasksReady,
+        tasksSleeping);
   }
 
-  private void fByteCount(final String name, double value) {
-    String suffix = "bytes";
+  private void sshSummary() {
+    IoAcceptor acceptor = daemon.getIoAcceptor();
+    if (acceptor == null) {
+      return;
+    }
+
+    long now = System.currentTimeMillis();
+    Collection<IoSession> list = acceptor.getManagedSessions().values();
+    long oldest = now;
+    double writeThroughput = 0.0;
+    for (IoSession s : list) {
+      oldest = Math.min(oldest, s.getCreationTime());
+    }
+
+    p.format(
+        "SSH:   %4d  users, oldest session started %s ago\n",
+        list.size(),
+        uptime(now - oldest));
+  }
+
+  private void jvmSummary() {
+    OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
+    RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
+    p.format("JVM: %s %s %s\n",
+        runtimeBean.getVmVendor(),
+        runtimeBean.getVmName(),
+        runtimeBean.getVmVersion());
+    p.format("  on %s %s %s\n", "",
+        osBean.getName(),
+        osBean.getVersion(),
+        osBean.getArch());
+    try {
+      p.format("  running as %s on %s\n",
+          System.getProperty("user.name"),
+          InetAddress.getLocalHost().getHostName());
+    } catch (UnknownHostException e) {
+    }
+    p.format("  cwd  %s\n", path(new File(".").getAbsoluteFile().getParentFile()));
+    p.format("  site %s\n", path(sitePath));
+  }
+
+  private String path(File file) {
+    try {
+      return file.getCanonicalPath();
+    } catch (IOException err) {
+      return file.getAbsolutePath();
+    }
+  }
+
+  private String uptime(long uptimeMillis) {
+    if (uptimeMillis < 1000) {
+      return String.format("%3d ms", uptimeMillis);
+    }
+
+    long uptime = uptimeMillis / 1000L;
+
+    long min = uptime / 60;
+    if (min < 60) {
+      return String.format("%2d min %2d sec", min, uptime - min * 60);
+    }
+
+    long hr = uptime / 3600;
+    if (hr < 24) {
+      min = (uptime - hr * 3600) / 60;
+      return String.format("%2d hrs %2d min", hr, min);
+    }
+
+    long days = uptime / (24 * 3600);
+    hr = (uptime - (days * 24 * 3600)) / 3600;
+    return String.format("%4d days %2d hrs", days, hr);
+  }
+
+  private String bytes(double value) {
+    value /= 1024;
+    String suffix = "k";
+
     if (value > 1024) {
       value /= 1024;
-      suffix = "kb";
+      suffix = "m";
     }
     if (value > 1024) {
       value /= 1024;
-      suffix = "mb";
+      suffix = "g";
     }
-    if (value > 1024) {
-      value /= 1024;
-      suffix = "gb";
-    }
-    p.println(String.format("  %1$-12s: %2$6.2f %3$s", name, value, suffix));
+    return String.format("%1$6.2f%2$s", value, suffix);
   }
 
   private String count(long cnt) {
@@ -211,9 +374,4 @@ final class ShowCaches extends CacheCommand {
     final long pcent = (100 * value) / total;
     return String.format("%3d%%", (int) pcent);
   }
-
-  private void fPercent(final String name, final long value, final long total) {
-    final long pcent = 0 < total ? (100 * value) / total : 0;
-    p.println(String.format("  %1$-12s: %2$3d%%", name, (int) pcent));
-  }
 }