Add new persistent cache "diff_file_path_list" to cache file paths only

The new cache caches list of paths of changed files. Main intention of
the new cache is to speed up change reindexing. By caching list of
changed files we avoid computing diff for that purpose.

NOTE: The existing "diff" cache implicitly caches list of changed files
but its entries are quite large to keep it populated for all changes.

Change-Id: I45f90deeba9fe70f32e4b0a5657bfb211ecc2eeb

Documentation/config-gerrit.txt

@@ -611,6 +611,7 @@ Default is 1024 for most caches, except:
 * `"adv_bases"`: default is `4096`
 * `"diff"`: default is `10m` (10 MiB of memory)
 * `"diff_intraline"`: default is `10m` (10 MiB of memory)
+* `"diff_file_list"`: default is `10m` (10 MiB of memory)
 * `"plugin_resources"`: default is 2m (2 MiB of memory)
 
 +
@@ -626,7 +627,10 @@ accessed order until the cache fits within this limit.  Caches may
 grow larger than this during the day, as the size check is only
 performed once every 24 hours.
 +
-Default is 128 MiB per cache.
+Default is 128 MiB per cache, except:
++
+* `"diff_file_list"`: default is `1g` (1 GiB of disk space)
+
 +
 If 0, disk storage for the cache is disabled.
 
@@ -698,6 +702,16 @@ estimate in bytes of memory used. Administrators should try to target
 cache.diff.memoryLimit to fit all files users will view in a 1 or 2
 day span.
 
+cache `"diff_file_list"`::
++
+Each item caches list of file paths which are different between two
+commits. Gerrit uses this cache to accelerate computing of the list
+of paths of changed files.
++
+Ideally, disk limit of this cache is large enough to cover all changes.
+This should significantly speed up change reindexing, especially
+full offline reindexing.
+
 cache `"git_tags"`::
 +
 If branch or reference level READ access controls are used, this

gerrit-server/src/main/java/com/google/gerrit/server/patch/FileList.java

@@ -0,0 +1,65 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.patch;
+
+import static com.google.gerrit.server.ioutil.BasicSerialization.readBytes;
+import static com.google.gerrit.server.ioutil.BasicSerialization.readString;
+import static com.google.gerrit.server.ioutil.BasicSerialization.writeBytes;
+import static com.google.gerrit.server.ioutil.BasicSerialization.writeString;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.zip.DeflaterOutputStream;
+import java.util.zip.InflaterInputStream;
+
+public class FileList implements Serializable {
+  private static final long serialVersionUID = PatchListKey.serialVersionUID;
+
+  private transient String[] paths;
+
+  public FileList(String[] paths) {
+    this.paths = paths;
+  }
+
+  public List<String> getPaths() {
+    return Collections.unmodifiableList(Arrays.asList(paths));
+  }
+
+  private void writeObject(ObjectOutputStream output) throws IOException {
+    ByteArrayOutputStream buf = new ByteArrayOutputStream();
+    try (DeflaterOutputStream out = new DeflaterOutputStream(buf)) {
+      writeString(out, Joiner.on('\n').join(paths));
+    }
+    writeBytes(output, buf.toByteArray());
+  }
+
+  private void readObject(ObjectInputStream input) throws IOException {
+    ByteArrayInputStream buf = new ByteArrayInputStream(readBytes(input));
+    try (InflaterInputStream in = new InflaterInputStream(buf)) {
+      List<String> l = Splitter.on('\n').splitToList(readString(in));
+      paths = l.toArray(new String[l.size()]);
+    }
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/patch/FileListLoader.java

@@ -0,0 +1,80 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.patch;
+
+import com.google.gerrit.reviewdb.client.Patch;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+public class FileListLoader implements Callable<FileList> {
+  static final Logger log = LoggerFactory.getLogger(FileListLoader.class);
+
+  public interface Factory {
+    FileListLoader create(PatchListKey key, Project.NameKey project);
+  }
+
+  private final PatchListCache patchListCache;
+  private final PatchListKey key;
+  private final Project.NameKey project;
+
+  @AssistedInject
+  FileListLoader(PatchListCache plc,
+      @Assisted PatchListKey k,
+      @Assisted Project.NameKey p) {
+    patchListCache = plc;
+    key = k;
+    project = p;
+  }
+
+  @Override
+  public FileList call() throws Exception {
+    PatchList patchList = patchListCache.get(key, project);
+    return toFileList(patchList);
+  }
+
+  static FileList toFileList(PatchList patchList) {
+    List<String> r = new ArrayList<>(patchList.getPatches().size());
+    for (PatchListEntry e : patchList.getPatches()) {
+      if (Patch.isMagic(e.getNewName())) {
+        continue;
+      }
+      switch (e.getChangeType()) {
+        case ADDED:
+        case MODIFIED:
+        case DELETED:
+        case COPIED:
+        case REWRITE:
+          r.add(e.getNewName());
+          break;
+
+        case RENAMED:
+          r.add(e.getOldName());
+          r.add(e.getNewName());
+          break;
+      }
+    }
+    Collections.sort(r);
+    return new FileList(r.toArray(new String[r.size()]));
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/patch/FileListWeigher.java

@@ -0,0 +1,35 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.patch;
+
+import com.google.common.cache.Weigher;
+
+/** Computes memory usage for FileList in bytes of memory used */
+public class FileListWeigher implements Weigher<PatchListKey, FileList> {
+
+  @Override
+  public int weigh(PatchListKey key, FileList value) {
+    int size = 16 + 4 * 8 + 2 * 36; // Size of PatchListKey, 64 bit JVM
+
+    // Size of the list of paths ...
+    for (String p : value.getPaths()) {
+      size += p.length();
+    }
+    // ... plus new-line separators between paths
+    size += value.getPaths().size() - 1;
+
+    return size;
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/patch/PatchListCache.java

@@ -33,4 +33,7 @@ public interface PatchListCache {
 
   IntraLineDiff getIntraLineDiff(IntraLineDiffKey key,
       IntraLineDiffArgs args);
+
+  FileList getFileList(Change change, PatchSet patchSet)
+      throws PatchListNotAvailableException;
 }

gerrit-server/src/main/java/com/google/gerrit/server/patch/PatchListCacheImpl.java

@@ -15,6 +15,7 @@
 
 package com.google.gerrit.server.patch;
 
+import static com.google.gerrit.server.patch.FileListLoader.toFileList;
 import com.google.common.cache.Cache;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.google.gerrit.extensions.client.DiffPreferencesInfo.Whitespace;
@@ -39,6 +40,7 @@ import java.util.concurrent.ExecutionException;
 public class PatchListCacheImpl implements PatchListCache {
   static final String FILE_NAME = "diff";
   static final String INTRA_NAME = "diff_intraline";
+  static final String FILE_LIST = "diff_file_list";
 
   public static Module module() {
     return new CacheModule() {
@@ -54,6 +56,12 @@ public class PatchListCacheImpl implements PatchListCache {
             .maximumWeight(10 << 20)
             .weigher(IntraLineWeigher.class);
 
+        factory(FileListLoader.Factory.class);
+        persist(FILE_LIST, PatchListKey.class, FileList.class)
+            .maximumWeight(10 << 20)
+            .weigher(FileListWeigher.class)
+            .diskLimit(1 << 30);
+
         bind(PatchListCacheImpl.class);
         bind(PatchListCache.class).to(PatchListCacheImpl.class);
       }
@@ -62,21 +70,27 @@ public class PatchListCacheImpl implements PatchListCache {
 
   private final Cache<PatchListKey, PatchList> fileCache;
   private final Cache<IntraLineDiffKey, IntraLineDiff> intraCache;
+  private final Cache<PatchListKey, FileList> fileListCache;
   private final PatchListLoader.Factory fileLoaderFactory;
   private final IntraLineLoader.Factory intraLoaderFactory;
+  private final FileListLoader.Factory fileListLoaderFactory;
   private final boolean computeIntraline;
 
   @Inject
   PatchListCacheImpl(
       @Named(FILE_NAME) Cache<PatchListKey, PatchList> fileCache,
       @Named(INTRA_NAME) Cache<IntraLineDiffKey, IntraLineDiff> intraCache,
+      @Named(FILE_LIST) Cache<PatchListKey, FileList> fileListCache,
       PatchListLoader.Factory fileLoaderFactory,
       IntraLineLoader.Factory intraLoaderFactory,
+      FileListLoader.Factory fileListLoaderFactory,
       @GerritServerConfig Config cfg) {
     this.fileCache = fileCache;
     this.intraCache = intraCache;
+    this.fileListCache = fileListCache;
     this.fileLoaderFactory = fileLoaderFactory;
     this.intraLoaderFactory = intraLoaderFactory;
+    this.fileListLoaderFactory = fileListLoaderFactory;
 
     this.computeIntraline =
         cfg.getBoolean("cache", INTRA_NAME, "enabled",
@@ -87,7 +101,9 @@ public class PatchListCacheImpl implements PatchListCache {
   public PatchList get(PatchListKey key, Project.NameKey project)
       throws PatchListNotAvailableException {
     try {
-      return fileCache.get(key, fileLoaderFactory.create(key, project));
+      PatchList pl = fileCache.get(key, fileLoaderFactory.create(key, project));
+      fileListCache.put(key, toFileList(pl));
+      return pl;
     } catch (ExecutionException e) {
       PatchListLoader.log.warn("Error computing " + key, e);
       throw new PatchListNotAvailableException(e);
@@ -140,4 +156,30 @@ public class PatchListCacheImpl implements PatchListCache {
     }
     return new IntraLineDiff(IntraLineDiff.Status.DISABLED);
   }
-}
+
+  @Override
+  public FileList getFileList(Change change, PatchSet patchSet)
+      throws PatchListNotAvailableException {
+    Project.NameKey project = change.getProject();
+    ObjectId b = ObjectId.fromString(patchSet.getRevision().get());
+    Whitespace ws = Whitespace.IGNORE_NONE;
+    return getFileList(PatchListKey.againstDefaultBase(b, ws), project);
+  }
+
+  private FileList getFileList(PatchListKey key, Project.NameKey project)
+      throws PatchListNotAvailableException {
+    try {
+      return fileListCache.get(key,
+          fileListLoaderFactory.create(key, project));
+    } catch (ExecutionException e) {
+      PatchListLoader.log.warn("Error computing " + key, e);
+      throw new PatchListNotAvailableException(e);
+    } catch (UncheckedExecutionException e) {
+      if (e.getCause() instanceof LargeObjectException) {
+        PatchListLoader.log.warn("Error computing " + key, e);
+        throw new PatchListNotAvailableException(e);
+      }
+      throw e;
+    }
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/query/change/ChangeData.java

@@ -38,7 +38,6 @@ import com.google.gerrit.reviewdb.client.Account;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.reviewdb.client.ChangeMessage;
 import com.google.gerrit.reviewdb.client.Comment;
-import com.google.gerrit.reviewdb.client.Patch;
 import com.google.gerrit.reviewdb.client.PatchSet;
 import com.google.gerrit.reviewdb.client.PatchSetApproval;
 import com.google.gerrit.reviewdb.client.Project;
@@ -59,9 +58,9 @@ import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.git.MergeUtil;
 import com.google.gerrit.server.notedb.ChangeNotes;
 import com.google.gerrit.server.notedb.NotesMigration;
+import com.google.gerrit.server.patch.FileList;
 import com.google.gerrit.server.patch.PatchList;
 import com.google.gerrit.server.patch.PatchListCache;
-import com.google.gerrit.server.patch.PatchListEntry;
 import com.google.gerrit.server.patch.PatchListNotAvailableException;
 import com.google.gerrit.server.project.ChangeControl;
 import com.google.gerrit.server.project.NoSuchChangeException;
@@ -337,6 +336,7 @@ public class ChangeData {
   private List<PatchSetApproval> currentApprovals;
   private Map<Integer, List<String>> files;
   private Map<Integer, Optional<PatchList>> patchLists;
+  private Map<Integer, Optional<FileList>> fileLists;
   private Collection<Comment> publishedComments;
   private CurrentUser visibleTo;
   private ChangeControl changeControl;
@@ -589,7 +589,7 @@ public class ChangeData {
         return null;
       }
 
-      Optional<PatchList> p = getPatchList(c, ps);
+      Optional<FileList> p = getFileList(c, ps);
       if (!p.isPresent()) {
         List<String> emptyFileList = Collections.emptyList();
         if (lazyLoad) {
@@ -598,28 +598,7 @@ public class ChangeData {
         return emptyFileList;
       }
 
-      r = new ArrayList<>(p.get().getPatches().size());
-      for (PatchListEntry e : p.get().getPatches()) {
-        if (Patch.isMagic(e.getNewName())) {
-          continue;
-        }
-        switch (e.getChangeType()) {
-          case ADDED:
-          case MODIFIED:
-          case DELETED:
-          case COPIED:
-          case REWRITE:
-            r.add(e.getNewName());
-            break;
-
-          case RENAMED:
-            r.add(e.getOldName());
-            r.add(e.getNewName());
-            break;
-        }
-      }
-      Collections.sort(r);
-      r = Collections.unmodifiableList(r);
+      r = p.get().getPaths();
       files.put(psId, r);
     }
     return r;
@@ -645,6 +624,26 @@ public class ChangeData {
     return r;
   }
 
+  private Optional<FileList> getFileList(Change c, PatchSet ps) {
+    Integer psId = ps.getId().get();
+    if (fileLists == null) {
+      fileLists = new HashMap<>();
+    }
+    Optional<FileList> r = fileLists.get(psId);
+    if (r == null) {
+      if (!lazyLoad) {
+        return Optional.absent();
+      }
+      try {
+        r = Optional.of(patchListCache.getFileList(c, ps));
+      } catch (PatchListNotAvailableException e) {
+        r = Optional.absent();
+      }
+      fileLists.put(psId, r);
+    }
+    return r;
+  }
+
   private Optional<ChangedLines> computeChangedLines() throws OrmException {
     Change c = change();
     if (c == null) {