Merge changes I45f90dee,I9f8c85d7

* changes:
  Add new persistent cache "diff_file_path_list" to cache file paths only
  Support for setting diskLimit from CacheModule bindings

Documentation/config-gerrit.txt

@@ -611,6 +611,7 @@ Default is 1024 for most caches, except:
 * `"adv_bases"`: default is `4096`
 * `"diff"`: default is `10m` (10 MiB of memory)
 * `"diff_intraline"`: default is `10m` (10 MiB of memory)
+* `"diff_file_list"`: default is `10m` (10 MiB of memory)
 * `"plugin_resources"`: default is 2m (2 MiB of memory)
 
 +
@@ -626,7 +627,10 @@ accessed order until the cache fits within this limit.  Caches may
 grow larger than this during the day, as the size check is only
 performed once every 24 hours.
 +
-Default is 128 MiB per cache.
+Default is 128 MiB per cache, except:
++
+* `"diff_file_list"`: default is `1g` (1 GiB of disk space)
+
 +
 If 0, disk storage for the cache is disabled.
 
@@ -698,6 +702,16 @@ estimate in bytes of memory used. Administrators should try to target
 cache.diff.memoryLimit to fit all files users will view in a 1 or 2
 day span.
 
+cache `"diff_file_list"`::
++
+Each item caches list of file paths which are different between two
+commits. Gerrit uses this cache to accelerate computing of the list
+of paths of changed files.
++
+Ideally, disk limit of this cache is large enough to cover all changes.
+This should significantly speed up change reindexing, especially
+full offline reindexing.
+
 cache `"git_tags"`::
 +
 If branch or reference level READ access controls are used, this

gerrit-cache-h2/src/main/java/com/google/gerrit/server/cache/h2/H2CacheFactory.java

@@ -178,7 +178,7 @@ class H2CacheFactory implements PersistentCacheFactory, LifecycleListener {
   public <K, V> LoadingCache<K, V> build(
       CacheBinding<K, V> def,
       CacheLoader<K, V> loader) {
-    long limit = config.getLong("cache", def.name(), "diskLimit", 128 << 20);
+    long limit = config.getLong("cache", def.name(), "diskLimit", def.diskLimit());
 
     if (cacheDir == null || limit <= 0) {
       return defaultFactory.build(def, loader);

gerrit-server/src/main/java/com/google/gerrit/server/cache/CacheBinding.java

@@ -26,6 +26,9 @@ public interface CacheBinding<K, V> {
   /** Set the total size of the cache. */
   CacheBinding<K, V> maximumWeight(long weight);
 
+  /** Set the total on-disk limit of the cache */
+  CacheBinding<K, V> diskLimit(long limit);
+
   /** Set the time an element lives before being expired. */
   CacheBinding<K, V> expireAfterWrite(long duration, TimeUnit durationUnits);
 
@@ -39,6 +42,7 @@ public interface CacheBinding<K, V> {
   TypeLiteral<K> keyType();
   TypeLiteral<V> valueType();
   long maximumWeight();
+  long diskLimit();
   @Nullable Long expireAfterWrite(TimeUnit unit);
   @Nullable Weigher<K, V> weigher();
   @Nullable CacheLoader<K, V> loader();

gerrit-server/src/main/java/com/google/gerrit/server/cache/CacheProvider.java

@@ -38,6 +38,7 @@ class CacheProvider<K, V>
   private final TypeLiteral<V> valType;
   private boolean persist;
   private long maximumWeight;
+  private long diskLimit;
   private Long expireAfterWrite;
   private Provider<CacheLoader<K, V>> loader;
   private Provider<Weigher<K, V>> weigher;
@@ -85,6 +86,15 @@ class CacheProvider<K, V>
     return this;
   }
 
+  @Override
+  public CacheBinding<K, V> diskLimit(long limit) {
+    Preconditions.checkState(!frozen, "binding frozen, cannot be modified");
+    Preconditions.checkState(persist,
+        "diskLimit supported for persistent caches only");
+    diskLimit = limit;
+    return this;
+  }
+
   @Override
   public CacheBinding<K, V> expireAfterWrite(long duration, TimeUnit unit) {
     Preconditions.checkState(!frozen, "binding frozen, cannot be modified");
@@ -129,6 +139,14 @@ class CacheProvider<K, V>
     return maximumWeight;
   }
 
+  @Override
+  public long diskLimit() {
+    if (diskLimit > 0) {
+      return diskLimit;
+    }
+    return 128 << 20;
+  }
+
   @Override
   @Nullable
   public Long expireAfterWrite(TimeUnit unit) {

gerrit-server/src/main/java/com/google/gerrit/server/patch/FileList.java

@@ -0,0 +1,65 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.patch;
+
+import static com.google.gerrit.server.ioutil.BasicSerialization.readBytes;
+import static com.google.gerrit.server.ioutil.BasicSerialization.readString;
+import static com.google.gerrit.server.ioutil.BasicSerialization.writeBytes;
+import static com.google.gerrit.server.ioutil.BasicSerialization.writeString;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.zip.DeflaterOutputStream;
+import java.util.zip.InflaterInputStream;
+
+public class FileList implements Serializable {
+  private static final long serialVersionUID = PatchListKey.serialVersionUID;
+
+  private transient String[] paths;
+
+  public FileList(String[] paths) {
+    this.paths = paths;
+  }
+
+  public List<String> getPaths() {
+    return Collections.unmodifiableList(Arrays.asList(paths));
+  }
+
+  private void writeObject(ObjectOutputStream output) throws IOException {
+    ByteArrayOutputStream buf = new ByteArrayOutputStream();
+    try (DeflaterOutputStream out = new DeflaterOutputStream(buf)) {
+      writeString(out, Joiner.on('\n').join(paths));
+    }
+    writeBytes(output, buf.toByteArray());
+  }
+
+  private void readObject(ObjectInputStream input) throws IOException {
+    ByteArrayInputStream buf = new ByteArrayInputStream(readBytes(input));
+    try (InflaterInputStream in = new InflaterInputStream(buf)) {
+      List<String> l = Splitter.on('\n').splitToList(readString(in));
+      paths = l.toArray(new String[l.size()]);
+    }
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/patch/FileListLoader.java

@@ -0,0 +1,80 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.patch;
+
+import com.google.gerrit.reviewdb.client.Patch;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+public class FileListLoader implements Callable<FileList> {
+  static final Logger log = LoggerFactory.getLogger(FileListLoader.class);
+
+  public interface Factory {
+    FileListLoader create(PatchListKey key, Project.NameKey project);
+  }
+
+  private final PatchListCache patchListCache;
+  private final PatchListKey key;
+  private final Project.NameKey project;
+
+  @AssistedInject
+  FileListLoader(PatchListCache plc,
+      @Assisted PatchListKey k,
+      @Assisted Project.NameKey p) {
+    patchListCache = plc;
+    key = k;
+    project = p;
+  }
+
+  @Override
+  public FileList call() throws Exception {
+    PatchList patchList = patchListCache.get(key, project);
+    return toFileList(patchList);
+  }
+
+  static FileList toFileList(PatchList patchList) {
+    List<String> r = new ArrayList<>(patchList.getPatches().size());
+    for (PatchListEntry e : patchList.getPatches()) {
+      if (Patch.isMagic(e.getNewName())) {
+        continue;
+      }
+      switch (e.getChangeType()) {
+        case ADDED:
+        case MODIFIED:
+        case DELETED:
+        case COPIED:
+        case REWRITE:
+          r.add(e.getNewName());
+          break;
+
+        case RENAMED:
+          r.add(e.getOldName());
+          r.add(e.getNewName());
+          break;
+      }
+    }
+    Collections.sort(r);
+    return new FileList(r.toArray(new String[r.size()]));
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/patch/FileListWeigher.java

@@ -0,0 +1,35 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.patch;
+
+import com.google.common.cache.Weigher;
+
+/** Computes memory usage for FileList in bytes of memory used */
+public class FileListWeigher implements Weigher<PatchListKey, FileList> {
+
+  @Override
+  public int weigh(PatchListKey key, FileList value) {
+    int size = 16 + 4 * 8 + 2 * 36; // Size of PatchListKey, 64 bit JVM
+
+    // Size of the list of paths ...
+    for (String p : value.getPaths()) {
+      size += p.length();
+    }
+    // ... plus new-line separators between paths
+    size += value.getPaths().size() - 1;
+
+    return size;
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/patch/PatchListCache.java

@@ -33,4 +33,7 @@ public interface PatchListCache {
 
   IntraLineDiff getIntraLineDiff(IntraLineDiffKey key,
       IntraLineDiffArgs args);
+
+  FileList getFileList(Change change, PatchSet patchSet)
+      throws PatchListNotAvailableException;
 }

gerrit-server/src/main/java/com/google/gerrit/server/patch/PatchListCacheImpl.java

@@ -15,6 +15,7 @@
 
 package com.google.gerrit.server.patch;
 
+import static com.google.gerrit.server.patch.FileListLoader.toFileList;
 import com.google.common.cache.Cache;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.google.gerrit.extensions.client.DiffPreferencesInfo.Whitespace;
@@ -39,6 +40,7 @@ import java.util.concurrent.ExecutionException;
 public class PatchListCacheImpl implements PatchListCache {
   static final String FILE_NAME = "diff";
   static final String INTRA_NAME = "diff_intraline";
+  static final String FILE_LIST = "diff_file_list";
 
   public static Module module() {
     return new CacheModule() {
@@ -54,6 +56,12 @@ public class PatchListCacheImpl implements PatchListCache {
             .maximumWeight(10 << 20)
             .weigher(IntraLineWeigher.class);
 
+        factory(FileListLoader.Factory.class);
+        persist(FILE_LIST, PatchListKey.class, FileList.class)
+            .maximumWeight(10 << 20)
+            .weigher(FileListWeigher.class)
+            .diskLimit(1 << 30);
+
         bind(PatchListCacheImpl.class);
         bind(PatchListCache.class).to(PatchListCacheImpl.class);
       }
@@ -62,21 +70,27 @@ public class PatchListCacheImpl implements PatchListCache {
 
   private final Cache<PatchListKey, PatchList> fileCache;
   private final Cache<IntraLineDiffKey, IntraLineDiff> intraCache;
+  private final Cache<PatchListKey, FileList> fileListCache;
   private final PatchListLoader.Factory fileLoaderFactory;
   private final IntraLineLoader.Factory intraLoaderFactory;
+  private final FileListLoader.Factory fileListLoaderFactory;
   private final boolean computeIntraline;
 
   @Inject
   PatchListCacheImpl(
       @Named(FILE_NAME) Cache<PatchListKey, PatchList> fileCache,
       @Named(INTRA_NAME) Cache<IntraLineDiffKey, IntraLineDiff> intraCache,
+      @Named(FILE_LIST) Cache<PatchListKey, FileList> fileListCache,
       PatchListLoader.Factory fileLoaderFactory,
       IntraLineLoader.Factory intraLoaderFactory,
+      FileListLoader.Factory fileListLoaderFactory,
       @GerritServerConfig Config cfg) {
     this.fileCache = fileCache;
     this.intraCache = intraCache;
+    this.fileListCache = fileListCache;
     this.fileLoaderFactory = fileLoaderFactory;
     this.intraLoaderFactory = intraLoaderFactory;
+    this.fileListLoaderFactory = fileListLoaderFactory;
 
     this.computeIntraline =
         cfg.getBoolean("cache", INTRA_NAME, "enabled",
@@ -87,7 +101,9 @@ public class PatchListCacheImpl implements PatchListCache {
   public PatchList get(PatchListKey key, Project.NameKey project)
       throws PatchListNotAvailableException {
     try {
-      return fileCache.get(key, fileLoaderFactory.create(key, project));
+      PatchList pl = fileCache.get(key, fileLoaderFactory.create(key, project));
+      fileListCache.put(key, toFileList(pl));
+      return pl;
     } catch (ExecutionException e) {
       PatchListLoader.log.warn("Error computing " + key, e);
       throw new PatchListNotAvailableException(e);
@@ -140,4 +156,30 @@ public class PatchListCacheImpl implements PatchListCache {
     }
     return new IntraLineDiff(IntraLineDiff.Status.DISABLED);
   }
-}
+
+  @Override
+  public FileList getFileList(Change change, PatchSet patchSet)
+      throws PatchListNotAvailableException {
+    Project.NameKey project = change.getProject();
+    ObjectId b = ObjectId.fromString(patchSet.getRevision().get());
+    Whitespace ws = Whitespace.IGNORE_NONE;
+    return getFileList(PatchListKey.againstDefaultBase(b, ws), project);
+  }
+
+  private FileList getFileList(PatchListKey key, Project.NameKey project)
+      throws PatchListNotAvailableException {
+    try {
+      return fileListCache.get(key,
+          fileListLoaderFactory.create(key, project));
+    } catch (ExecutionException e) {
+      PatchListLoader.log.warn("Error computing " + key, e);
+      throw new PatchListNotAvailableException(e);
+    } catch (UncheckedExecutionException e) {
+      if (e.getCause() instanceof LargeObjectException) {
+        PatchListLoader.log.warn("Error computing " + key, e);
+        throw new PatchListNotAvailableException(e);
+      }
+      throw e;
+    }
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/query/change/ChangeData.java

@@ -38,7 +38,6 @@ import com.google.gerrit.reviewdb.client.Account;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.reviewdb.client.ChangeMessage;
 import com.google.gerrit.reviewdb.client.Comment;
-import com.google.gerrit.reviewdb.client.Patch;
 import com.google.gerrit.reviewdb.client.PatchSet;
 import com.google.gerrit.reviewdb.client.PatchSetApproval;
 import com.google.gerrit.reviewdb.client.Project;
@@ -59,9 +58,9 @@ import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.git.MergeUtil;
 import com.google.gerrit.server.notedb.ChangeNotes;
 import com.google.gerrit.server.notedb.NotesMigration;
+import com.google.gerrit.server.patch.FileList;
 import com.google.gerrit.server.patch.PatchList;
 import com.google.gerrit.server.patch.PatchListCache;
-import com.google.gerrit.server.patch.PatchListEntry;
 import com.google.gerrit.server.patch.PatchListNotAvailableException;
 import com.google.gerrit.server.project.ChangeControl;
 import com.google.gerrit.server.project.NoSuchChangeException;
@@ -337,6 +336,7 @@ public class ChangeData {
   private List<PatchSetApproval> currentApprovals;
   private Map<Integer, List<String>> files;
   private Map<Integer, Optional<PatchList>> patchLists;
+  private Map<Integer, Optional<FileList>> fileLists;
   private Collection<Comment> publishedComments;
   private CurrentUser visibleTo;
   private ChangeControl changeControl;
@@ -589,7 +589,7 @@ public class ChangeData {
         return null;
       }
 
-      Optional<PatchList> p = getPatchList(c, ps);
+      Optional<FileList> p = getFileList(c, ps);
       if (!p.isPresent()) {
         List<String> emptyFileList = Collections.emptyList();
         if (lazyLoad) {
@@ -598,28 +598,7 @@ public class ChangeData {
         return emptyFileList;
       }
 
-      r = new ArrayList<>(p.get().getPatches().size());
-      for (PatchListEntry e : p.get().getPatches()) {
-        if (Patch.isMagic(e.getNewName())) {
-          continue;
-        }
-        switch (e.getChangeType()) {
-          case ADDED:
-          case MODIFIED:
-          case DELETED:
-          case COPIED:
-          case REWRITE:
-            r.add(e.getNewName());
-            break;
-
-          case RENAMED:
-            r.add(e.getOldName());
-            r.add(e.getNewName());
-            break;
-        }
-      }
-      Collections.sort(r);
-      r = Collections.unmodifiableList(r);
+      r = p.get().getPaths();
       files.put(psId, r);
     }
     return r;
@@ -645,6 +624,26 @@ public class ChangeData {
     return r;
   }
 
+  private Optional<FileList> getFileList(Change c, PatchSet ps) {
+    Integer psId = ps.getId().get();
+    if (fileLists == null) {
+      fileLists = new HashMap<>();
+    }
+    Optional<FileList> r = fileLists.get(psId);
+    if (r == null) {
+      if (!lazyLoad) {
+        return Optional.absent();
+      }
+      try {
+        r = Optional.of(patchListCache.getFileList(c, ps));
+      } catch (PatchListNotAvailableException e) {
+        r = Optional.absent();
+      }
+      fileLists.put(psId, r);
+    }
+    return r;
+  }
+
   private Optional<ChangedLines> computeChangedLines() throws OrmException {
     Change c = change();
     if (c == null) {