Cache JGit FileHeader and EditList inside of Ehcache

We now cache the FileHeader and EditList information for a given
file difference within an Ehcache, which is a memory and disk based
cache and claims to be faster than streaming this information out
of our local SQL database.  This uses up a lot less disk space,
as we store only the edit list and not the full patch content.

Signed-off-by: Shawn O. Pearce <sop@google.com>

src/main/java/com/google/gerrit/server/GerritServer.java

@@ -38,6 +38,7 @@ import com.google.gerrit.client.workflow.SubmitFunction;
 import com.google.gerrit.git.MergeQueue;
 import com.google.gerrit.git.RepositoryCache;
 import com.google.gerrit.git.WorkQueue;
+import com.google.gerrit.server.patch.DiffCacheEntryFactory;
 import com.google.gwtjsonrpc.server.SignedToken;
 import com.google.gwtjsonrpc.server.XsrfException;
 import com.google.gwtorm.client.OrmException;
@@ -51,6 +52,7 @@ import net.sf.ehcache.Ehcache;
 import net.sf.ehcache.config.CacheConfiguration;
 import net.sf.ehcache.config.Configuration;
 import net.sf.ehcache.config.DiskStoreConfiguration;
+import net.sf.ehcache.constructs.blocking.SelfPopulatingCache;
 import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
 
 import org.apache.commons.codec.binary.Base64;
@@ -171,6 +173,7 @@ public class GerritServer {
   private final SignedToken emailReg;
   private final RepositoryCache repositories;
   private final javax.mail.Session outgoingMail;
+  private final SelfPopulatingCache diffCache;
 
   private GerritServer() throws OrmException, XsrfException {
     db = createDatabase();
@@ -228,6 +231,7 @@ public class GerritServer {
     Common.setGroupCache(new GroupCache(sConfig));
 
     cacheMgr = new CacheManager(createCacheConfiguration());
+    diffCache = startCacheDiff();
   }
 
   private Configuration createCacheConfiguration() {
@@ -242,6 +246,7 @@ public class GerritServer {
       mgrCfg.addCache(c);
     }
 
+    mgrCfg.addCache(configureNamedCache(mgrCfg, "diff", true, 0));
     return mgrCfg;
   }
 
@@ -325,6 +330,15 @@ public class GerritServer {
     return cfg;
   }
 
+  private SelfPopulatingCache startCacheDiff() {
+    final Cache dc = cacheMgr.getCache("diff");
+    final SelfPopulatingCache r;
+
+    r = new SelfPopulatingCache(dc, new DiffCacheEntryFactory(this));
+    cacheMgr.replaceCacheWithDecoratedCache(dc, r);
+    return r;
+  }
+
   private Database<ReviewDb> createDatabase() throws OrmException {
     final String dsName = "java:comp/env/jdbc/ReviewDb";
     try {
@@ -760,6 +774,11 @@ public class GerritServer {
     return cacheMgr.getCache(name);
   }
 
+  /** Get the self-populating cache of DiffCacheContent entities. */
+  public SelfPopulatingCache getDiffCache() {
+    return diffCache;
+  }
+
   /** The mail session used to send messages; null if not configured. */
   public javax.mail.Session getOutgoingMail() {
     return outgoingMail;

src/main/java/com/google/gerrit/server/patch/DiffCacheContent.java

@@ -0,0 +1,218 @@
+// Copyright (C) 2009 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.patch;
+
+import org.spearce.jgit.diff.Edit;
+import org.spearce.jgit.errors.IncorrectObjectTypeException;
+import org.spearce.jgit.errors.MissingObjectException;
+import org.spearce.jgit.lib.AbbreviatedObjectId;
+import org.spearce.jgit.lib.Constants;
+import org.spearce.jgit.lib.ObjectId;
+import org.spearce.jgit.lib.Repository;
+import org.spearce.jgit.patch.CombinedFileHeader;
+import org.spearce.jgit.patch.FileHeader;
+import org.spearce.jgit.patch.Patch;
+import org.spearce.jgit.revwalk.RevTree;
+import org.spearce.jgit.revwalk.RevWalk;
+import org.spearce.jgit.treewalk.TreeWalk;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public final class DiffCacheContent implements Serializable {
+  private static final long serialVersionUID = 1L;
+
+  public static DiffCacheContent create(final Repository db,
+      final DiffCacheKey key, final FileHeader file)
+      throws MissingObjectException, IncorrectObjectTypeException, IOException {
+    final ObjectId o;
+    final ObjectId n;
+
+    if (file.getOldId() != null && file.getOldId().isComplete()) {
+      o = toId(file.getOldId());
+    } else {
+      String path = key.getSourceFileName();
+      if (path == null) {
+        path = key.getFileName();
+      }
+      o = find(db, key.getOldId(), path);
+    }
+
+    if (file.getNewId() != null && file.getNewId().isComplete()) {
+      n = toId(file.getNewId());
+    } else {
+      n = find(db, key.getNewId(), key.getFileName());
+    }
+
+    return new DiffCacheContent(file, o, n);
+  }
+
+  private static ObjectId toId(final AbbreviatedObjectId a) {
+    final ObjectId o = a.toObjectId();
+    return ObjectId.zeroId().equals(o) ? null : o;
+  }
+
+  static ObjectId find(final Repository db, final ObjectId treeIsh,
+      final String path) throws MissingObjectException,
+      IncorrectObjectTypeException, IOException {
+    final RevTree tree = new RevWalk(db).parseTree(treeIsh);
+    final TreeWalk tw = TreeWalk.forPath(db, path, tree);
+    if (tw == null) {
+      return null;
+    }
+    return tw.getObjectId(0);
+  }
+
+  private transient boolean noDifference;
+  private transient ObjectId oldId;
+  private transient ObjectId newId;
+  private transient FileHeader header;
+  private transient List<Edit> edits;
+
+  public DiffCacheContent() {
+    noDifference = true;
+  }
+
+  private DiffCacheContent(final FileHeader h, final ObjectId o,
+      final ObjectId n) {
+    noDifference = false;
+    header = compact(h);
+    oldId = o;
+    newId = n;
+
+    if (h instanceof CombinedFileHeader || h.getHunks().isEmpty()) {
+      edits = Collections.emptyList();
+    } else {
+      edits = h.toEditList();
+    }
+  }
+
+  public boolean isNoDifference() {
+    return noDifference;
+  }
+
+  public ObjectId getOldId() {
+    return oldId;
+  }
+
+  public ObjectId getNewId() {
+    return newId;
+  }
+
+  public FileHeader getFileHeader() {
+    return header;
+  }
+
+  public List<Edit> getEdits() {
+    return edits;
+  }
+
+  private void writeObject(final ObjectOutputStream out) throws IOException {
+    out.writeBoolean(noDifference);
+    if (noDifference) {
+      return;
+    }
+
+    writeId(out, oldId);
+    writeId(out, newId);
+
+    final int len = end(header) - header.getStartOffset();
+    out.writeInt(len);
+    out.write(header.getBuffer(), header.getStartOffset(), len);
+
+    out.writeInt(edits.size());
+    for (final Edit e : edits) {
+      out.writeInt(e.getBeginA());
+      out.writeInt(e.getEndA());
+      out.writeInt(e.getBeginB());
+      out.writeInt(e.getEndB());
+    }
+  }
+
+  private static void writeId(final ObjectOutputStream out, final ObjectId o)
+      throws IOException {
+    final byte[] idBuf = new byte[Constants.OBJECT_ID_LENGTH];
+    out.writeBoolean(o != null);
+    if (o != null) {
+      o.copyRawTo(idBuf, 0);
+      out.write(idBuf);
+    }
+  }
+
+  private void readObject(final ObjectInputStream in) throws IOException {
+    noDifference = in.readBoolean();
+    if (noDifference) {
+      return;
+    }
+
+    oldId = readId(in);
+    newId = readId(in);
+
+    final byte[] buf = new byte[in.readInt()];
+    in.readFully(buf);
+    header = parse(buf);
+
+    final int editCount = in.readInt();
+    edits = new ArrayList<Edit>(editCount);
+    for (int n = editCount - 1; 0 <= n; n--) {
+      final int beginA = in.readInt();
+      final int endA = in.readInt();
+      final int beginB = in.readInt();
+      final int endB = in.readInt();
+      edits.add(new Edit(beginA, endA, beginB, endB));
+    }
+  }
+
+  private static ObjectId readId(final ObjectInputStream in) throws IOException {
+    if (in.readBoolean()) {
+      final byte[] idBuf = new byte[Constants.OBJECT_ID_LENGTH];
+      in.readFully(idBuf);
+      return ObjectId.fromRaw(idBuf);
+    }
+    return null;
+  }
+
+  private static FileHeader parse(final byte[] buf) {
+    final Patch p = new Patch();
+    p.parse(buf, 0, buf.length);
+    return p.getFiles().get(0);
+  }
+
+  private static FileHeader compact(final FileHeader h) {
+    final int end = end(h);
+    if (h.getStartOffset() == 0 && end == h.getBuffer().length) {
+      return h;
+    }
+
+    final byte[] buf = new byte[end - h.getStartOffset()];
+    System.arraycopy(h.getBuffer(), h.getStartOffset(), buf, 0, buf.length);
+    return parse(buf);
+  }
+
+  private static int end(final FileHeader h) {
+    if (h instanceof CombinedFileHeader) {
+      return h.getEndOffset();
+    }
+    if (!h.getHunks().isEmpty()) {
+      return h.getHunks().get(0).getStartOffset();
+    }
+    return h.getEndOffset();
+  }
+}

src/main/java/com/google/gerrit/server/patch/DiffCacheEntryFactory.java

@@ -0,0 +1,97 @@
+// Copyright (C) 2009 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.patch;
+
+import com.google.gerrit.client.reviewdb.Project;
+import com.google.gerrit.git.InvalidRepositoryException;
+import com.google.gerrit.server.GerritServer;
+
+import net.sf.ehcache.constructs.blocking.CacheEntryFactory;
+
+import org.spearce.jgit.lib.Repository;
+import org.spearce.jgit.patch.FileHeader;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class DiffCacheEntryFactory implements CacheEntryFactory {
+  private final GerritServer server;
+
+  public DiffCacheEntryFactory(final GerritServer gs) {
+    server = gs;
+  }
+
+  public Object createEntry(Object genericKey) throws Exception {
+    final DiffCacheKey key = (DiffCacheKey) genericKey;
+    final Repository db = open(key.getProjectKey());
+    final List<String> args = new ArrayList<String>();
+    args.add("git");
+    args.add("--git-dir=.");
+    args.add("diff-tree");
+    if (key.getSourceFileName() != null) {
+      args.add("-M");
+    }
+    args.add("--full-index");
+
+    if (key.getOldId() == null) {
+      args.add("--cc");
+    } else {
+      args.add("--unified=1");
+      args.add(key.getOldId().name());
+    }
+    args.add(key.getNewId().name());
+    args.add("--");
+    args.add(key.getFileName());
+    if (key.getSourceFileName() != null) {
+      args.add(key.getSourceFileName());
+    }
+
+    final Process proc =
+        Runtime.getRuntime().exec(args.toArray(new String[args.size()]), null,
+            db.getDirectory());
+    final FileHeader file;
+    try {
+      final org.spearce.jgit.patch.Patch p = new org.spearce.jgit.patch.Patch();
+      proc.getOutputStream().close();
+      proc.getErrorStream().close();
+      p.parse(proc.getInputStream());
+      proc.getInputStream().close();
+
+      if (p.getFiles().isEmpty()) {
+        return new DiffCacheContent();
+
+      } else if (p.getFiles().size() != 1) {
+        throw new IOException("unexpected file count: " + key);
+      }
+
+      file = p.getFiles().get(0);
+    } finally {
+      try {
+        if (proc.waitFor() != 0) {
+          throw new IOException("git diff-tree exited abnormally: " + key);
+        }
+      } catch (InterruptedException ie) {
+      }
+    }
+
+    return DiffCacheContent.create(db, key, file);
+  }
+
+  private Repository open(final Project.NameKey key)
+      throws InvalidRepositoryException {
+    return server.getRepositoryCache().get(key.get());
+  }
+}

src/main/java/com/google/gerrit/server/patch/DiffCacheKey.java

@@ -0,0 +1,186 @@
+// Copyright (C) 2009 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.patch;
+
+import com.google.gerrit.client.reviewdb.Patch;
+import com.google.gerrit.client.reviewdb.Project;
+
+import org.spearce.jgit.lib.AnyObjectId;
+import org.spearce.jgit.lib.Constants;
+import org.spearce.jgit.lib.ObjectId;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+public final class DiffCacheKey implements Serializable {
+  private static final long serialVersionUID = 1L;
+
+  private transient Project.NameKey projectKey;
+  private transient ObjectId oldId;
+  private transient ObjectId newId;
+  private transient String fileName;
+  private transient String sourceFileName;
+
+  public DiffCacheKey(final Project.NameKey pnk, final AnyObjectId a,
+      final AnyObjectId b, final Patch p) {
+    this(pnk, a, b, p.getFileName(), p.getSourceFileName());
+  }
+
+  public DiffCacheKey(final Project.NameKey p, final AnyObjectId a,
+      final AnyObjectId b, final String dname, final String sname) {
+    projectKey = p;
+    oldId = a != null ? a.copy() : null;
+    newId = b.copy();
+    fileName = dname;
+    sourceFileName = sname;
+  }
+
+  public Project.NameKey getProjectKey() {
+    return projectKey;
+  }
+
+  public ObjectId getOldId() {
+    return oldId;
+  }
+
+  public ObjectId getNewId() {
+    return newId;
+  }
+
+  public String getFileName() {
+    return fileName;
+  }
+
+  public String getSourceFileName() {
+    return sourceFileName;
+  }
+
+  @Override
+  public int hashCode() {
+    int h = projectKey.hashCode();
+
+    if (oldId != null) {
+      h *= 31;
+      h += oldId.hashCode();
+    }
+
+    h *= 31;
+    h += newId.hashCode();
+
+    h *= 31;
+    h += fileName.hashCode();
+
+    if (sourceFileName != null) {
+      h *= 31;
+      h += sourceFileName.hashCode();
+    }
+
+    return h;
+  }
+
+  @Override
+  public boolean equals(final Object o) {
+    if (o instanceof DiffCacheKey) {
+      final DiffCacheKey k = (DiffCacheKey) o;
+      return projectKey.equals(k.projectKey) && eq(oldId, k.oldId)
+          && eq(newId, k.newId) && eq(fileName, k.fileName)
+          && eq(sourceFileName, k.sourceFileName);
+    }
+    return false;
+  }
+
+  private static boolean eq(final ObjectId a, final ObjectId b) {
+    if (a == null && b == null) {
+      return true;
+    }
+    return a != null && b != null ? a.equals(b) : false;
+  }
+
+  private static boolean eq(final String a, final String b) {
+    if (a == null && b == null) {
+      return true;
+    }
+    return a != null && a.equals(b);
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder r = new StringBuilder();
+    r.append("DiffCache[");
+    r.append(projectKey.toString());
+    r.append(" ");
+    if (oldId != null) {
+      r.append(oldId.name());
+      r.append("..");
+    }
+    r.append(newId.name());
+    r.append(" -- ");
+    r.append(fileName);
+    if (sourceFileName != null) {
+      r.append(" ");
+      r.append(sourceFileName);
+    }
+    r.append("]");
+    return r.toString();
+  }
+
+  private void writeObject(final ObjectOutputStream out) throws IOException {
+    final byte[] idBuf = new byte[Constants.OBJECT_ID_LENGTH];
+
+    out.writeUTF(projectKey.get());
+
+    out.writeBoolean(oldId != null);
+    if (oldId != null) {
+      oldId.copyRawTo(idBuf, 0);
+      out.write(idBuf);
+    }
+
+    newId.copyRawTo(idBuf, 0);
+    out.write(idBuf);
+
+    writeString(out, fileName);
+    writeString(out, sourceFileName);
+  }
+
+  private void readObject(final ObjectInputStream in) throws IOException {
+    final byte[] idBuf = new byte[Constants.OBJECT_ID_LENGTH];
+
+    projectKey = new Project.NameKey(in.readUTF());
+
+    if (in.readBoolean()) {
+      in.readFully(idBuf);
+      oldId = ObjectId.fromRaw(idBuf);
+    }
+
+    in.readFully(idBuf);
+    newId = ObjectId.fromRaw(idBuf);
+
+    fileName = readString(in);
+    sourceFileName = readString(in);
+  }
+
+  private static void writeString(final ObjectOutputStream out, final String s)
+      throws IOException {
+    out.writeUTF(s != null ? s : "");
+  }
+
+  private static String readString(final ObjectInputStream in)
+      throws IOException {
+    final String s = in.readUTF();
+    return s.length() > 0 ? s : null;
+  }
+}

src/main/java/com/google/gerrit/server/ssh/AdminFlushCaches.java

@@ -34,6 +34,15 @@ class AdminFlushCaches extends AbstractCommand {
       if (Common.getGerritConfig().getLoginType() == LoginType.OPENID) {
         flushCache("openid");
       }
+
+      try {
+        getGerritServer().getDiffCache().flush();
+      } catch (Throwable e1) {
+        try {
+          err.write(("warning: " + err.toString()).getBytes("UTF-8"));
+        } catch (IOException e2) {
+        }
+      }
     } else {
       throw new Failure(1, "fatal: Not a Gerrit administrator");
     }