Merge changes If8ea827f,Ic56a37b7,I154aba00,I0c6fc4e0

* changes:
  Extract the keys for the (Git)ModifiedFilesCaches to separate classes
  Add protobuf serializer for the ModifiedFilesCache key
  Add protobuf serializers for the GitModifiedFilesCache entities
  Add the GitModifiedFilesCache and ModifiedFilesCache
This commit is contained in:
Youssef Elghareeb
2020-10-28 13:38:17 +00:00
committed by Gerrit Code Review
19 changed files with 1193 additions and 0 deletions

View File

@@ -49,6 +49,7 @@ Apache2.0
* commons:compress
* commons:dbcp
* commons:lang
* commons:lang3
* commons:net
* commons:pool
* commons:validator

View File

@@ -113,6 +113,7 @@ java_library(
"//lib/commons:compress",
"//lib/commons:dbcp",
"//lib/commons:lang",
"//lib/commons:lang3",
"//lib/commons:net",
"//lib/commons:validator",
"//lib/errorprone:annotations",

View File

@@ -0,0 +1,28 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch;
import com.google.gerrit.server.patch.diff.ModifiedFilesCache;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCache;
/**
* Thrown by the diff caches - the {@link GitModifiedFilesCache} and the {@link ModifiedFilesCache},
* if the implementations failed to retrieve the modified files between the 2 commits.
*/
public class DiffNotAvailableException extends Exception {
public DiffNotAvailableException(Throwable cause) {
super(cause);
}
}

View File

@@ -0,0 +1,79 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package com.google.gerrit.server.patch;
import com.google.gerrit.server.patch.diff.ModifiedFilesCache;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCache;
import java.io.IOException;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
/**
* A utility class used by the diff cache interfaces {@link GitModifiedFilesCache} and {@link
* ModifiedFilesCache}.
*/
public class DiffUtil {
/**
* Returns the Git tree object ID pointed to by the commitId parameter.
*
* @param rw a {@link RevWalk} of an opened repository that is used to walk the commit graph.
* @param commitId 20 bytes commitId SHA-1 hash.
* @return Git tree object ID pointed to by the commitId.
*/
public static ObjectId getTreeId(RevWalk rw, ObjectId commitId) throws IOException {
RevCommit current = rw.parseCommit(commitId);
return current.getTree().getId();
}
/**
* Returns the RevCommit object given the 20 bytes commitId SHA-1 hash.
*
* @param rw a {@link RevWalk} of an opened repository that is used to walk the commit graph.
* @param commitId 20 bytes commitId SHA-1 hash
* @return The RevCommit representing the commit in Git
* @throws IOException a pack file or loose object could not be read while parsing the commits.
*/
public static RevCommit getRevCommit(RevWalk rw, ObjectId commitId) throws IOException {
return rw.parseCommit(commitId);
}
/**
* Returns true if the commitA and commitB parameters are parent/child, if they have a common
* parent, or if any of them is a root or merge commit.
*/
public static boolean areRelated(RevCommit commitA, RevCommit commitB) {
return commitA == null
|| isRootOrMergeCommit(commitA)
|| isRootOrMergeCommit(commitB)
|| areParentAndChild(commitA, commitB)
|| haveCommonParent(commitA, commitB);
}
private static boolean isRootOrMergeCommit(RevCommit commit) {
return commit.getParentCount() != 1;
}
private static boolean areParentAndChild(RevCommit commitA, RevCommit commitB) {
return ObjectId.isEqual(commitA.getParent(0), commitB)
|| ObjectId.isEqual(commitB.getParent(0), commitA);
}
private static boolean haveCommonParent(RevCommit commitA, RevCommit commitB) {
return ObjectId.isEqual(commitA.getParent(0), commitB.getParent(0));
}
}

View File

@@ -0,0 +1,45 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch.diff;
import com.google.common.collect.ImmutableList;
import com.google.gerrit.server.patch.DiffNotAvailableException;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCache;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCacheImpl;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCacheKey;
import com.google.gerrit.server.patch.gitdiff.ModifiedFile;
/**
* A cache for the list of Git modified files between 2 commits (patchsets) with extra Gerrit logic.
*
* <p>The loader uses the underlying {@link GitModifiedFilesCacheImpl} to retrieve the git modified
* files.
*
* <p>If the {@link ModifiedFilesCacheKey#aCommit()} is equal to {@link
* org.eclipse.jgit.lib.Constants#EMPTY_TREE_ID}, the diff will be evaluated against the empty tree,
* and the result will be exactly the same as the caller can get from {@link
* GitModifiedFilesCache#get(GitModifiedFilesCacheKey)}
*/
public interface ModifiedFilesCache {
/**
* @param key used to identify two git commits and contains other attributes to control the diff
* calculation.
* @return the list of {@link ModifiedFile}s between the 2 git commits identified by the key.
* @throws DiffNotAvailableException the supplied commits IDs of the key do no exist, are not IDs
* of a commit, or an exception occurred while reading a pack file.
*/
ImmutableList<ModifiedFile> get(ModifiedFilesCacheKey key) throws DiffNotAvailableException;
}

View File

@@ -0,0 +1,206 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch.diff;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static org.eclipse.jgit.lib.Constants.EMPTY_TREE_ID;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.collect.Streams;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.patch.DiffNotAvailableException;
import com.google.gerrit.server.patch.DiffUtil;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCache;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCacheImpl;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCacheKey;
import com.google.gerrit.server.patch.gitdiff.ModifiedFile;
import com.google.inject.Inject;
import com.google.inject.Module;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
import java.io.IOException;
import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
/**
* A cache for the list of Git modified files between 2 commits (patchsets) with extra Gerrit logic.
*
* <p>The loader of this cache wraps a {@link GitModifiedFilesCache} to retrieve the git modified
* files.
*
* <p>If the {@link ModifiedFilesCacheKey#aCommit()} is equal to {@link
* org.eclipse.jgit.lib.Constants#EMPTY_TREE_ID}, the diff will be evaluated against the empty tree,
* and the result will be exactly the same as the caller can get from {@link
* GitModifiedFilesCache#get(GitModifiedFilesCacheKey)}
*/
public class ModifiedFilesCacheImpl implements ModifiedFilesCache {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final String MODIFIED_FILES = "modified_files";
private final LoadingCache<ModifiedFilesCacheKey, ImmutableList<ModifiedFile>> cache;
public static Module module() {
return new CacheModule() {
@Override
protected void configure() {
bind(ModifiedFilesCache.class).to(ModifiedFilesCacheImpl.class);
// The documentation has some defaults and recommendations for setting the cache
// attributes:
// https://gerrit-review.googlesource.com/Documentation/config-gerrit.html#cache.
// The cache is using the default disk limit as per section cache.<name>.diskLimit
// in the cache documentation link.
persist(
ModifiedFilesCacheImpl.MODIFIED_FILES,
ModifiedFilesCacheKey.class,
new TypeLiteral<ImmutableList<ModifiedFile>>() {})
.keySerializer(ModifiedFilesCacheKey.Serializer.INSTANCE)
.valueSerializer(GitModifiedFilesCacheImpl.ValueSerializer.INSTANCE)
.maximumWeight(10 << 20)
.weigher(ModifiedFilesWeigher.class)
.version(1)
.loader(ModifiedFilesLoader.class);
}
};
}
@Inject
public ModifiedFilesCacheImpl(
@Named(ModifiedFilesCacheImpl.MODIFIED_FILES)
LoadingCache<ModifiedFilesCacheKey, ImmutableList<ModifiedFile>> cache) {
this.cache = cache;
}
@Override
public ImmutableList<ModifiedFile> get(ModifiedFilesCacheKey key)
throws DiffNotAvailableException {
try {
return cache.get(key);
} catch (Exception e) {
throw new DiffNotAvailableException(e);
}
}
static class ModifiedFilesLoader
extends CacheLoader<ModifiedFilesCacheKey, ImmutableList<ModifiedFile>> {
private final GitModifiedFilesCache gitCache;
private final GitRepositoryManager repoManager;
@Inject
ModifiedFilesLoader(GitModifiedFilesCache gitCache, GitRepositoryManager repoManager) {
this.gitCache = gitCache;
this.repoManager = repoManager;
}
@Override
public ImmutableList<ModifiedFile> load(ModifiedFilesCacheKey key)
throws IOException, DiffNotAvailableException {
try (Repository repo = repoManager.openRepository(key.project());
RevWalk rw = new RevWalk(repo.newObjectReader())) {
return loadModifiedFiles(key, rw);
}
}
private ImmutableList<ModifiedFile> loadModifiedFiles(ModifiedFilesCacheKey key, RevWalk rw)
throws IOException, DiffNotAvailableException {
ObjectId aTree =
key.aCommit().equals(EMPTY_TREE_ID)
? key.aCommit()
: DiffUtil.getTreeId(rw, key.aCommit());
ObjectId bTree = DiffUtil.getTreeId(rw, key.bCommit());
GitModifiedFilesCacheKey gitKey =
GitModifiedFilesCacheKey.builder()
.project(key.project())
.aTree(aTree)
.bTree(bTree)
.renameScore(key.renameScore())
.build();
List<ModifiedFile> modifiedFiles = gitCache.get(gitKey);
if (key.aCommit().equals(EMPTY_TREE_ID)) {
return ImmutableList.copyOf(modifiedFiles);
}
RevCommit revCommitA = DiffUtil.getRevCommit(rw, key.aCommit());
RevCommit revCommitB = DiffUtil.getRevCommit(rw, key.bCommit());
if (DiffUtil.areRelated(revCommitA, revCommitB)) {
return ImmutableList.copyOf(modifiedFiles);
}
Set<String> touchedFiles =
getTouchedFilesWithParents(
key, revCommitA.getParent(0).getId(), revCommitB.getParent(0).getId(), rw);
return modifiedFiles.stream()
.filter(f -> isTouched(touchedFiles, f))
.collect(toImmutableList());
}
/**
* Returns the paths of files that were modified between the old and new commits versus their
* parents (i.e. old commit vs. its parent, and new commit vs. its parent).
*
* @param key the {@link ModifiedFilesCacheKey} representing the commits we are diffing
* @param rw a {@link RevWalk} for the repository
* @return The list of modified files between the old/new commits and their parents
*/
private Set<String> getTouchedFilesWithParents(
ModifiedFilesCacheKey key, ObjectId parentOfA, ObjectId parentOfB, RevWalk rw)
throws IOException {
try {
// TODO(ghareeb): as an enhancement: the 3 calls of the underlying git cache can be combined
GitModifiedFilesCacheKey oldVsBaseKey =
GitModifiedFilesCacheKey.create(
key.project(), parentOfA, key.aCommit(), key.renameScore(), rw);
List<ModifiedFile> oldVsBase = gitCache.get(oldVsBaseKey);
GitModifiedFilesCacheKey newVsBaseKey =
GitModifiedFilesCacheKey.create(
key.project(), parentOfB, key.bCommit(), key.renameScore(), rw);
List<ModifiedFile> newVsBase = gitCache.get(newVsBaseKey);
return Sets.union(getOldAndNewPaths(oldVsBase), getOldAndNewPaths(newVsBase));
} catch (DiffNotAvailableException e) {
logger.atWarning().log(
"Failed to retrieve the touched files' commits (%s, %s) and parents (%s, %s): %s",
key.aCommit(), key.bCommit(), parentOfA, parentOfB, e.getMessage());
return ImmutableSet.of();
}
}
private ImmutableSet<String> getOldAndNewPaths(List<ModifiedFile> files) {
return files.stream()
.flatMap(
file -> Stream.concat(Streams.stream(file.oldPath()), Streams.stream(file.newPath())))
.collect(ImmutableSet.toImmutableSet());
}
private static boolean isTouched(Set<String> touchedFilePaths, ModifiedFile modifiedFile) {
String oldFilePath = modifiedFile.oldPath().orElse(null);
String newFilePath = modifiedFile.newPath().orElse(null);
// One of the above file paths could be /dev/null but we need not explicitly check for this
// value as the set of file paths shouldn't contain it.
return touchedFilePaths.contains(oldFilePath) || touchedFilePaths.contains(newFilePath);
}
}
}

View File

@@ -0,0 +1,116 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch.diff;
import com.google.auto.value.AutoValue;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.proto.Protos;
import com.google.gerrit.server.cache.proto.Cache.ModifiedFilesKeyProto;
import com.google.gerrit.server.cache.serialize.CacheSerializer;
import com.google.gerrit.server.cache.serialize.ObjectIdConverter;
import org.eclipse.jgit.lib.ObjectId;
/** Cache key for the {@link com.google.gerrit.server.patch.diff.ModifiedFilesCache} */
@AutoValue
public abstract class ModifiedFilesCacheKey {
/** A specific git project / repository. */
public abstract Project.NameKey project();
/** @return the old commit ID used in the git tree diff */
public abstract ObjectId aCommit();
/** @return the new commit ID used in the git tree diff */
public abstract ObjectId bCommit();
/**
* Percentage score used to identify a file as a "rename". A special value of -1 means that the
* computation will ignore renames and rename detection will be disabled.
*/
public abstract int renameScore();
public boolean renameDetectionEnabled() {
return renameScore() != -1;
}
/** Returns the size of the object in bytes */
public int weight() {
return stringSize(project().get()) // project
+ 20 * 2 // aCommit and bCommit
+ 4; // renameScore
}
public static ModifiedFilesCacheKey.Builder builder() {
return new AutoValue_ModifiedFilesCacheKey.Builder();
}
@AutoValue.Builder
public abstract static class Builder {
public abstract ModifiedFilesCacheKey.Builder project(NameKey value);
public abstract ModifiedFilesCacheKey.Builder aCommit(ObjectId value);
public abstract ModifiedFilesCacheKey.Builder bCommit(ObjectId value);
public ModifiedFilesCacheKey.Builder disableRenameDetection() {
renameScore(-1);
return this;
}
public abstract ModifiedFilesCacheKey.Builder renameScore(int value);
public abstract ModifiedFilesCacheKey build();
}
public enum Serializer implements CacheSerializer<ModifiedFilesCacheKey> {
INSTANCE;
@Override
public byte[] serialize(ModifiedFilesCacheKey key) {
ObjectIdConverter idConverter = ObjectIdConverter.create();
return Protos.toByteArray(
ModifiedFilesKeyProto.newBuilder()
.setProject(key.project().get())
.setACommit(idConverter.toByteString(key.aCommit()))
.setBCommit(idConverter.toByteString(key.bCommit()))
.setRenameScore(key.renameScore())
.build());
}
@Override
public ModifiedFilesCacheKey deserialize(byte[] in) {
ModifiedFilesKeyProto proto = Protos.parseUnchecked(ModifiedFilesKeyProto.parser(), in);
ObjectIdConverter idConverter = ObjectIdConverter.create();
return ModifiedFilesCacheKey.builder()
.project(NameKey.parse(proto.getProject()))
.aCommit(idConverter.fromByteString(proto.getACommit()))
.bCommit(idConverter.fromByteString(proto.getBCommit()))
.renameScore(proto.getRenameScore())
.build();
}
}
private static int stringSize(String str) {
if (str != null) {
// each character in the string occupies 2 bytes. Ignoring the fixed overhead for the string
// (length, offset and hash code) since they are negligible and do not
// affect the comparison of 2 strings
return str.length() * 2;
}
return 0;
}
}

View File

@@ -0,0 +1,31 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch.diff;
import com.google.common.cache.Weigher;
import com.google.common.collect.ImmutableList;
import com.google.gerrit.server.patch.gitdiff.ModifiedFile;
public class ModifiedFilesWeigher
implements Weigher<ModifiedFilesCacheKey, ImmutableList<ModifiedFile>> {
@Override
public int weigh(ModifiedFilesCacheKey key, ImmutableList<ModifiedFile> modifiedFiles) {
int weight = key.weight();
for (ModifiedFile modifiedFile : modifiedFiles) {
weight += modifiedFile.weight();
}
return weight;
}
}

View File

@@ -0,0 +1,40 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch.gitdiff;
import com.google.common.collect.ImmutableList;
import com.google.gerrit.server.patch.DiffNotAvailableException;
import com.google.gerrit.server.patch.diff.ModifiedFilesCache;
/**
* A cache interface for identifying the list of Git modified files between 2 different git trees.
* This cache does not read the actual file contents, nor does it include the edits (modified
* regions) of the file.
*
* <p>The other {@link ModifiedFilesCache} is similar to this cache, and includes other extra Gerrit
* logic that we need to add with the list of modified files.
*/
public interface GitModifiedFilesCache {
/**
* Computes the list of of {@link ModifiedFile}s between the 2 git trees.
*
* @param key used to identify two git trees and contains other attributes to control the diff
* calculation.
* @return the list of {@link ModifiedFile}s between the 2 git trees identified by the key.
* @throws DiffNotAvailableException trees cannot be read or file contents cannot be read.
*/
ImmutableList<ModifiedFile> get(GitModifiedFilesCacheKey key) throws DiffNotAvailableException;
}

View File

@@ -0,0 +1,177 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch.gitdiff;
import static com.google.common.collect.ImmutableList.toImmutableList;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.entities.Patch;
import com.google.gerrit.proto.Protos;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.cache.proto.Cache.ModifiedFilesProto;
import com.google.gerrit.server.cache.serialize.CacheSerializer;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.patch.DiffNotAvailableException;
import com.google.inject.Inject;
import com.google.inject.Module;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.util.io.DisabledOutputStream;
/** Implementation of the {@link GitModifiedFilesCache} */
public class GitModifiedFilesCacheImpl implements GitModifiedFilesCache {
private static final String GIT_MODIFIED_FILES = "git_modified_files";
private static final ImmutableMap<ChangeType, Patch.ChangeType> changeTypeMap =
ImmutableMap.of(
DiffEntry.ChangeType.ADD,
Patch.ChangeType.ADDED,
DiffEntry.ChangeType.MODIFY,
Patch.ChangeType.MODIFIED,
DiffEntry.ChangeType.DELETE,
Patch.ChangeType.DELETED,
DiffEntry.ChangeType.RENAME,
Patch.ChangeType.RENAMED,
DiffEntry.ChangeType.COPY,
Patch.ChangeType.COPIED);
private LoadingCache<GitModifiedFilesCacheKey, ImmutableList<ModifiedFile>> cache;
public static Module module() {
return new CacheModule() {
@Override
protected void configure() {
bind(GitModifiedFilesCache.class).to(GitModifiedFilesCacheImpl.class);
persist(
GIT_MODIFIED_FILES,
GitModifiedFilesCacheKey.class,
new TypeLiteral<ImmutableList<ModifiedFile>>() {})
.keySerializer(GitModifiedFilesCacheKey.Serializer.INSTANCE)
.valueSerializer(ValueSerializer.INSTANCE)
// The documentation has some defaults and recommendations for setting the cache
// attributes:
// https://gerrit-review.googlesource.com/Documentation/config-gerrit.html#cache.
.maximumWeight(10 << 20)
.weigher(GitModifiedFilesWeigher.class)
// The cache is using the default disk limit as per section cache.<name>.diskLimit
// in the cache documentation link.
.version(1)
.loader(GitModifiedFilesCacheImpl.Loader.class);
}
};
}
@Inject
public GitModifiedFilesCacheImpl(
@Named(GIT_MODIFIED_FILES)
LoadingCache<GitModifiedFilesCacheKey, ImmutableList<ModifiedFile>> cache) {
this.cache = cache;
}
@Override
public ImmutableList<ModifiedFile> get(GitModifiedFilesCacheKey key)
throws DiffNotAvailableException {
try {
return cache.get(key);
} catch (ExecutionException e) {
throw new DiffNotAvailableException(e);
}
}
static class Loader extends CacheLoader<GitModifiedFilesCacheKey, ImmutableList<ModifiedFile>> {
private final GitRepositoryManager repoManager;
@Inject
Loader(GitRepositoryManager repoManager) {
this.repoManager = repoManager;
}
@Override
public ImmutableList<ModifiedFile> load(GitModifiedFilesCacheKey key) throws IOException {
try (Repository repo = repoManager.openRepository(key.project());
ObjectReader reader = repo.newObjectReader()) {
List<DiffEntry> entries = getGitTreeDiff(repo, reader, key);
return entries.stream().map(Loader::toModifiedFile).collect(toImmutableList());
}
}
private List<DiffEntry> getGitTreeDiff(
Repository repo, ObjectReader reader, GitModifiedFilesCacheKey key) throws IOException {
try (DiffFormatter df = new DiffFormatter(DisabledOutputStream.INSTANCE)) {
df.setReader(reader, repo.getConfig());
if (key.renameDetection()) {
df.setDetectRenames(true);
df.getRenameDetector().setRenameScore(key.renameScore());
}
// The scan method only returns the file paths that are different. Callers may choose to
// format these paths themselves.
return df.scan(key.aTree(), key.bTree());
}
}
private static ModifiedFile toModifiedFile(DiffEntry entry) {
String oldPath = entry.getOldPath();
String newPath = entry.getNewPath();
return ModifiedFile.builder()
.changeType(toChangeType(entry.getChangeType()))
.oldPath(oldPath.equals(DiffEntry.DEV_NULL) ? Optional.empty() : Optional.of(oldPath))
.newPath(newPath.equals(DiffEntry.DEV_NULL) ? Optional.empty() : Optional.of(newPath))
.build();
}
private static Patch.ChangeType toChangeType(DiffEntry.ChangeType changeType) {
if (!changeTypeMap.containsKey(changeType)) {
throw new IllegalArgumentException("Unsupported type " + changeType);
}
return changeTypeMap.get(changeType);
}
}
public enum ValueSerializer implements CacheSerializer<ImmutableList<ModifiedFile>> {
INSTANCE;
@Override
public byte[] serialize(ImmutableList<ModifiedFile> modifiedFiles) {
ModifiedFilesProto.Builder builder = ModifiedFilesProto.newBuilder();
modifiedFiles.forEach(
f -> builder.addModifiedFile(ModifiedFile.Serializer.INSTANCE.toProto(f)));
return Protos.toByteArray(builder.build());
}
@Override
public ImmutableList<ModifiedFile> deserialize(byte[] in) {
ImmutableList.Builder<ModifiedFile> modifiedFiles = ImmutableList.builder();
ModifiedFilesProto modifiedFilesProto =
Protos.parseUnchecked(ModifiedFilesProto.parser(), in);
modifiedFilesProto
.getModifiedFileList()
.forEach(f -> modifiedFiles.add(ModifiedFile.Serializer.INSTANCE.fromProto(f)));
return modifiedFiles.build();
}
}
}

View File

@@ -0,0 +1,137 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch.gitdiff;
import com.google.auto.value.AutoValue;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.proto.Protos;
import com.google.gerrit.server.cache.proto.Cache.GitModifiedFilesKeyProto;
import com.google.gerrit.server.cache.serialize.CacheSerializer;
import com.google.gerrit.server.cache.serialize.ObjectIdConverter;
import com.google.gerrit.server.patch.DiffUtil;
import java.io.IOException;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.revwalk.RevWalk;
/** Cache key for the {@link GitModifiedFilesCache}. */
@AutoValue
public abstract class GitModifiedFilesCacheKey {
/** A specific git project / repository. */
public abstract Project.NameKey project();
/**
* The git SHA-1 {@link ObjectId} of the first git tree object for which the diff should be
* computed.
*/
public abstract ObjectId aTree();
/**
* The git SHA-1 {@link ObjectId} of the second git tree object for which the diff should be
* computed.
*/
public abstract ObjectId bTree();
/**
* Percentage score used to identify a file as a rename. This value is only available if {@link
* #renameDetection()} is true. Otherwise, this method will return -1.
*
* <p>This value will be used to set the rename score of {@link
* org.eclipse.jgit.diff.DiffFormatter#getRenameDetector()}.
*/
public abstract int renameScore();
/** Returns true if rename detection was set for this key. */
public boolean renameDetection() {
return renameScore() != -1;
}
public static GitModifiedFilesCacheKey create(
Project.NameKey project, ObjectId aCommit, ObjectId bCommit, int renameScore, RevWalk rw)
throws IOException {
ObjectId aTree = DiffUtil.getTreeId(rw, aCommit);
ObjectId bTree = DiffUtil.getTreeId(rw, bCommit);
return builder().project(project).aTree(aTree).bTree(bTree).renameScore(renameScore).build();
}
public static Builder builder() {
return new AutoValue_GitModifiedFilesCacheKey.Builder();
}
/** Returns the size of the object in bytes */
public int weight() {
return stringSize(project().get())
+ 20 * 2 // old and new tree IDs
+ 4; // rename score
}
@AutoValue.Builder
public abstract static class Builder {
public abstract Builder project(NameKey value);
public abstract Builder aTree(ObjectId value);
public abstract Builder bTree(ObjectId value);
public abstract Builder renameScore(int value);
public Builder disableRenameDetection() {
renameScore(-1);
return this;
}
public abstract GitModifiedFilesCacheKey build();
}
public enum Serializer implements CacheSerializer<GitModifiedFilesCacheKey> {
INSTANCE;
@Override
public byte[] serialize(GitModifiedFilesCacheKey key) {
ObjectIdConverter idConverter = ObjectIdConverter.create();
return Protos.toByteArray(
GitModifiedFilesKeyProto.newBuilder()
.setProject(key.project().get())
.setATree(idConverter.toByteString(key.aTree()))
.setBTree(idConverter.toByteString(key.bTree()))
.setRenameScore(key.renameScore())
.build());
}
@Override
public GitModifiedFilesCacheKey deserialize(byte[] in) {
GitModifiedFilesKeyProto proto = Protos.parseUnchecked(GitModifiedFilesKeyProto.parser(), in);
ObjectIdConverter idConverter = ObjectIdConverter.create();
return GitModifiedFilesCacheKey.builder()
.project(NameKey.parse(proto.getProject()))
.aTree(idConverter.fromByteString(proto.getATree()))
.bTree(idConverter.fromByteString(proto.getBTree()))
.renameScore(proto.getRenameScore())
.build();
}
}
private static int stringSize(String str) {
if (str != null) {
// each character in the string occupies 2 bytes. Ignoring the fixed overhead for the string
// (length, offset and hash code) since they are negligible and do not
// affect the comparison of 2 strings
return str.length() * 2;
}
return 0;
}
}

View File

@@ -0,0 +1,26 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch.gitdiff;
import com.google.common.cache.Weigher;
import com.google.common.collect.ImmutableList;
public class GitModifiedFilesWeigher
implements Weigher<GitModifiedFilesCacheKey, ImmutableList<ModifiedFile>> {
@Override
public int weigh(GitModifiedFilesCacheKey key, ImmutableList<ModifiedFile> modifiedFiles) {
return key.weight() + modifiedFiles.stream().mapToInt(ModifiedFile::weight).sum();
}
}

View File

@@ -0,0 +1,123 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.patch.gitdiff;
import com.google.auto.value.AutoValue;
import com.google.gerrit.entities.Patch.ChangeType;
import com.google.gerrit.proto.Protos;
import com.google.gerrit.server.cache.proto.Cache.ModifiedFileProto;
import com.google.gerrit.server.cache.serialize.CacheSerializer;
import com.google.protobuf.Descriptors.FieldDescriptor;
import java.util.Optional;
/**
* An entity representing a Modified file due to a diff between 2 git trees. This entity contains
* the change type and the old & new paths, but does not include any actual content diff of the
* file.
*/
@AutoValue
public abstract class ModifiedFile {
/**
* Returns the change type (i.e. add, delete, modify, rename, etc...) associated with this
* modified file.
*/
public abstract ChangeType changeType();
/**
* Returns the old name associated with this file. An empty optional is returned if {@link
* #changeType()} is equal to {@link ChangeType#ADDED}.
*/
public abstract Optional<String> oldPath();
/**
* Returns the new name associated with this file. An empty optional is returned if {@link
* #changeType()} is equal to {@link ChangeType#DELETED}
*/
public abstract Optional<String> newPath();
public static Builder builder() {
return new AutoValue_ModifiedFile.Builder();
}
/** Computes this object's weight, which is its size in bytes. */
public int weight() {
int weight = 1; // the changeType field
if (oldPath().isPresent()) {
weight += oldPath().get().length();
}
if (newPath().isPresent()) {
weight += newPath().get().length();
}
return weight;
}
@AutoValue.Builder
public abstract static class Builder {
public abstract Builder changeType(ChangeType value);
public abstract Builder oldPath(Optional<String> value);
public abstract Builder newPath(Optional<String> value);
public abstract ModifiedFile build();
}
enum Serializer implements CacheSerializer<ModifiedFile> {
INSTANCE;
private static final FieldDescriptor oldPathDescriptor =
ModifiedFileProto.getDescriptor().findFieldByName("old_path");
private static final FieldDescriptor newPathDescriptor =
ModifiedFileProto.getDescriptor().findFieldByName("new_path");
@Override
public byte[] serialize(ModifiedFile modifiedFile) {
return Protos.toByteArray(toProto(modifiedFile));
}
public ModifiedFileProto toProto(ModifiedFile modifiedFile) {
ModifiedFileProto.Builder builder = ModifiedFileProto.newBuilder();
builder.setChangeType(modifiedFile.changeType().toString());
if (modifiedFile.oldPath().isPresent()) {
builder.setOldPath(modifiedFile.oldPath().get());
}
if (modifiedFile.newPath().isPresent()) {
builder.setNewPath(modifiedFile.newPath().get());
}
return builder.build();
}
@Override
public ModifiedFile deserialize(byte[] in) {
ModifiedFileProto modifiedFileProto = Protos.parseUnchecked(ModifiedFileProto.parser(), in);
return fromProto(modifiedFileProto);
}
public ModifiedFile fromProto(ModifiedFileProto modifiedFileProto) {
ModifiedFile.Builder builder = ModifiedFile.builder();
builder.changeType(ChangeType.valueOf(modifiedFileProto.getChangeType()));
if (modifiedFileProto.hasField(oldPathDescriptor)) {
builder.oldPath(Optional.of(modifiedFileProto.getOldPath()));
}
if (modifiedFileProto.hasField(newPathDescriptor)) {
builder.newPath(Optional.of(modifiedFileProto.getNewPath()));
}
return builder.build();
}
}
}

View File

@@ -118,6 +118,12 @@ public class RevisionDiffIT extends AbstractDaemonTest {
assertDiffForNewFile(result, COMMIT_MSG, result.getCommit().getFullMessage());
}
@Ignore
@Test
public void diffWithRootCommit() throws Exception {
// TODO(ghareeb): Implement this test
}
@Test
public void patchsetLevelFileDiffIsEmpty() throws Exception {
PushOneCommit.Result result = createChange();

View File

@@ -7,6 +7,7 @@ junit_tests(
"//java/com/google/gerrit/common:server",
"//java/com/google/gerrit/entities",
"//java/com/google/gerrit/extensions:api",
"//java/com/google/gerrit/server",
"//java/com/google/gerrit/server/cache/serialize",
"//java/com/google/gerrit/server/cache/serialize/entities",
"//java/com/google/gerrit/server/cache/testing",

View File

@@ -0,0 +1,43 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.cache.serialize.entities;
import static com.google.common.truth.Truth.assertThat;
import com.google.gerrit.entities.Project;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCacheKey;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCacheKey.Serializer;
import org.eclipse.jgit.lib.ObjectId;
import org.junit.Test;
public class GitModifiedFilesCacheKeySerializerTest {
private static final ObjectId TREE_ID_1 =
ObjectId.fromString("123e9fa8a286255ac7d5ba11b598892735758391");
private static final ObjectId TREE_ID_2 =
ObjectId.fromString("d07a03a9818c120301cb5b4a969b035479400b5f");
@Test
public void roundTrip() {
GitModifiedFilesCacheKey key =
GitModifiedFilesCacheKey.builder()
.project(Project.NameKey.parse("Project/X"))
.aTree(TREE_ID_1)
.bTree(TREE_ID_2)
.renameScore(65)
.build();
byte[] serialized = Serializer.INSTANCE.serialize(key);
assertThat(Serializer.INSTANCE.deserialize(serialized)).isEqualTo(key);
}
}

View File

@@ -0,0 +1,42 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.cache.serialize.entities;
import static com.google.common.truth.Truth.assertThat;
import com.google.gerrit.entities.Project;
import com.google.gerrit.server.patch.diff.ModifiedFilesCacheKey;
import org.eclipse.jgit.lib.ObjectId;
import org.junit.Test;
public class ModifiedFilesCacheKeySerializerTest {
private static final ObjectId COMMIT_ID_1 =
ObjectId.fromString("123e9fa8a286255ac7d5ba11b598892735758391");
private static final ObjectId COMMIT_ID_2 =
ObjectId.fromString("d07a03a9818c120301cb5b4a969b035479400b5f");
@Test
public void roundTrip() {
ModifiedFilesCacheKey key =
ModifiedFilesCacheKey.builder()
.project(Project.NameKey.parse("Project/X"))
.aCommit(COMMIT_ID_1)
.bCommit(COMMIT_ID_2)
.renameScore(65)
.build();
byte[] serialized = ModifiedFilesCacheKey.Serializer.INSTANCE.serialize(key);
assertThat(ModifiedFilesCacheKey.Serializer.INSTANCE.deserialize(serialized)).isEqualTo(key);
}
}

View File

@@ -0,0 +1,56 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.cache.serialize.entities;
import static com.google.common.truth.Truth.assertThat;
import com.google.common.collect.ImmutableList;
import com.google.gerrit.entities.Patch.ChangeType;
import com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCacheImpl.ValueSerializer;
import com.google.gerrit.server.patch.gitdiff.ModifiedFile;
import java.util.Optional;
import org.junit.Test;
public class ModifiedFilesSerializerTest {
@Test
public void roundTrip() {
ImmutableList.Builder<ModifiedFile> builder = ImmutableList.builder();
builder.add(
ModifiedFile.builder()
.changeType(ChangeType.DELETED)
.oldPath(Optional.of("file_1.txt"))
.newPath(Optional.of("file_2.txt"))
.build());
builder.add(
ModifiedFile.builder()
.changeType(ChangeType.ADDED)
.oldPath(Optional.empty())
.newPath(Optional.of("file_3.txt"))
.build());
// Note: the default value for strings in protocol buffers is the empty string, hence the
// serializer will not be able to differentiate between an empty optional and an optional
// with an empty string, i.e. if we serialize an optional with an empty string, the deserialized
// object will be an empty optional. That should not be problematic in this case because file
// paths cannot be empty anyway.
ImmutableList<ModifiedFile> modifiedFiles = builder.build();
byte[] serialized = ValueSerializer.INSTANCE.serialize(modifiedFiles);
assertThat(ValueSerializer.INSTANCE.deserialize(serialized)).isEqualTo(modifiedFiles);
}
}

View File

@@ -526,3 +526,38 @@ message AllCommentContextProto {
}
repeated CommentContextProto context = 1;
}
// Serialized key for
// com.google.gerrit.server.patch.gitdiff.GitModifiedFilesCacheKey
// Next ID: 5
message GitModifiedFilesKeyProto {
string project = 1;
bytes a_tree = 2; // SHA-1 hash of the left git tree ID in the diff
bytes b_tree = 3; // SHA-1 hash of the right git tree ID in the diff
int32 rename_score = 4;
}
// Serialized key for
// com.google.gerrit.server.patch.diff.ModifiedFilesCacheKey
// Next ID: 5
message ModifiedFilesKeyProto {
string project = 1;
bytes a_commit = 2; // SHA-1 hash of the left commit ID in the diff
bytes b_commit = 3; // SHA-1 hash of the right commit ID in the diff
int32 rename_score = 4;
}
// Serialized form of com.google.gerrit.server.patch.gitdiff.ModifiedFile
// Next ID: 4
message ModifiedFileProto {
string change_type = 1; // ENUM as string
string old_path = 2;
string new_path = 3;
}
// Serialized form of a collection of
// com.google.gerrit.server.patch.gitdiff.ModifiedFile
// Next ID: 2
message ModifiedFilesProto {
repeated ModifiedFileProto modifiedFile = 1;
}