opendev/gerrit
Code Issues Proposed changes

Merge changes from topic "tag-cache-proto"

Browse Source 
* changes:
  Serialize TagSetHolder with protobuf
  TagCache: Eliminate extra lock on cache load
  Extract StringSerializer to its own non-test class
This commit is contained in:
Dave Borowitz
2018-07-31 14:37:10 +00:00
committed by Gerrit Code Review
parent 94677b2b2f b30a414171
commit e071121c81
9 changed files with 508 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
java/com/google/gerrit/server/cache/StringSerializer.java vendored Normal file
View File

@@ -0,0 +1,70 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.cache;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.annotations.VisibleForTesting;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CodingErrorAction;
public enum StringSerializer implements CacheSerializer<String> {
INSTANCE;
@Override
public byte[] serialize(String object) {
return serialize(UTF_8, object);
}
@VisibleForTesting
static byte[] serialize(Charset charset, String s) {
if (s.isEmpty()) {
return new byte[0];
}
try {
ByteBuffer buf =
charset
.newEncoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT)
.encode(CharBuffer.wrap(s));
byte[] result = new byte[buf.remaining()];
buf.get(result);
return result;
} catch (CharacterCodingException e) {
throw new IllegalStateException("Failed to serialize string", e);
}
}
@Override
public String deserialize(byte[] in) {
if (in.length == 0) {
return "";
}
try {
return UTF_8
.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT)
.decode(ByteBuffer.wrap(in))
.toString();
} catch (CharacterCodingException e) {
throw new IllegalStateException("Failed to deserialize string", e);
}
}
}

74
java/com/google/gerrit/server/git/TagCache.java
View File

@@ -17,14 +17,12 @@ package com.google.gerrit.server.git;
import com.google.common.cache.Cache;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.cache.StringSerializer;
import com.google.inject.Inject;
import com.google.inject.Module;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.concurrent.ExecutionException;
import org.eclipse.jgit.lib.ObjectId;
@Singleton
@@ -35,17 +33,19 @@ public class TagCache {
return new CacheModule() {
@Override
protected void configure() {
persist(CACHE_NAME, String.class, EntryVal.class);
persist(CACHE_NAME, String.class, TagSetHolder.class)
.version(1)
.keySerializer(StringSerializer.INSTANCE)
.valueSerializer(TagSetHolder.Serializer.INSTANCE);
bind(TagCache.class);
}
};
}
private final Cache<String, EntryVal> cache;
private final Object createLock = new Object();
private final Cache<String, TagSetHolder> cache;
@Inject
TagCache(@Named(CACHE_NAME) Cache<String, EntryVal> cache) {
TagCache(@Named(CACHE_NAME) Cache<String, TagSetHolder> cache) {
this.cache = cache;
}
@@ -68,62 +68,26 @@ public class TagCache {
// never fail with an exception. Some of these references can be null
// (e.g. not all projects are cached, or the cache is not current).
//
EntryVal val = cache.getIfPresent(name.get());
if (val != null) {
TagSetHolder holder = val.holder;
if (holder != null) {
TagSet tags = holder.getTagSet();
if (tags != null) {
if (tags.updateFastForward(refName, oldValue, newValue)) {
cache.put(name.get(), val);
}
TagSetHolder holder = cache.getIfPresent(name.get());
if (holder != null) {
TagSet tags = holder.getTagSet();
if (tags != null) {
if (tags.updateFastForward(refName, oldValue, newValue)) {
cache.put(name.get(), holder);
}
}
}
}
public TagSetHolder get(Project.NameKey name) {
EntryVal val = cache.getIfPresent(name.get());
if (val == null) {
synchronized (createLock) {
val = cache.getIfPresent(name.get());
if (val == null) {
val = new EntryVal();
val.holder = new TagSetHolder(name);
cache.put(name.get(), val);
}
}
try {
return cache.get(name.get(), () -> new TagSetHolder(name));
} catch (ExecutionException e) {
throw new IllegalStateException(e);
}
return val.holder;
}
void put(Project.NameKey name, TagSetHolder tags) {
EntryVal val = new EntryVal();
val.holder = tags;
cache.put(name.get(), val);
}
public static class EntryVal implements Serializable {
static final long serialVersionUID = 1L;
transient TagSetHolder holder;
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
holder = new TagSetHolder(new Project.NameKey(in.readUTF()));
if (in.readBoolean()) {
TagSet tags = new TagSet(holder.getProjectName());
tags.readObject(in);
holder.setTagSet(tags);
}
}
private void writeObject(ObjectOutputStream out) throws IOException {
TagSet tags = holder.getTagSet();
out.writeUTF(holder.getProjectName().get());
out.writeBoolean(tags != null);
if (tags != null) {
tags.writeObject(out);
}
}
cache.put(name.get(), tags);
}
}

131
java/com/google/gerrit/server/git/TagSet.java
View File

@@ -14,16 +14,19 @@
package com.google.gerrit.server.git;
import static org.eclipse.jgit.lib.ObjectIdSerializer.readWithoutMarker;
import static org.eclipse.jgit.lib.ObjectIdSerializer.writeWithoutMarker;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.collect.Maps;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.server.cache.ProtoCacheSerializers.ObjectIdConverter;
import com.google.gerrit.server.cache.proto.Cache.TagSetHolderProto.TagSetProto;
import com.google.gerrit.server.cache.proto.Cache.TagSetHolderProto.TagSetProto.CachedRefProto;
import com.google.gerrit.server.cache.proto.Cache.TagSetHolderProto.TagSetProto.TagProto;
import com.google.protobuf.ByteString;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Map;
@@ -47,15 +50,35 @@ class TagSet {
private final ObjectIdOwnerMap<Tag> tags;
TagSet(Project.NameKey projectName) {
this(projectName, new HashMap<>(), new ObjectIdOwnerMap<>());
}
TagSet(Project.NameKey projectName, HashMap<String, CachedRef> refs, ObjectIdOwnerMap<Tag> tags) {
this.projectName = projectName;
this.refs = new HashMap<>();
this.tags = new ObjectIdOwnerMap<>();
this.refs = refs;
this.tags = tags;
}
Project.NameKey getProjectName() {
return projectName;
}
Tag lookupTag(AnyObjectId id) {
return tags.get(id);
}
// Test methods have obtuse names in addition to annotations, since they expose mutable state
// which would be easy to corrupt.
@VisibleForTesting
Map<String, CachedRef> getRefsForTesting() {
return refs;
}
@VisibleForTesting
ObjectIdOwnerMap<Tag> getTagsForTesting() {
return tags;
}
boolean updateFastForward(String refName, ObjectId oldValue, ObjectId newValue) {
CachedRef ref = refs.get(refName);
if (ref != null) {
@@ -191,36 +214,46 @@ class TagSet {
}
}
void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
int refCnt = in.readInt();
for (int i = 0; i < refCnt; i++) {
String name = in.readUTF();
int flag = in.readInt();
ObjectId id = readWithoutMarker(in);
refs.put(name, new CachedRef(flag, id));
}
static TagSet fromProto(TagSetProto proto) {
ObjectIdConverter idConverter = ObjectIdConverter.create();
int tagCnt = in.readInt();
for (int i = 0; i < tagCnt; i++) {
ObjectId id = readWithoutMarker(in);
BitSet flags = (BitSet) in.readObject();
tags.add(new Tag(id, flags));
}
HashMap<String, CachedRef> refs = Maps.newHashMapWithExpectedSize(proto.getRefCount());
proto
.getRefMap()
.forEach(
(n, cr) ->
refs.put(n, new CachedRef(cr.getFlag(), idConverter.fromByteString(cr.getId()))));
ObjectIdOwnerMap<Tag> tags = new ObjectIdOwnerMap<>();
proto
.getTagList()
.forEach(
t ->
tags.add(
new Tag(
idConverter.fromByteString(t.getId()),
BitSet.valueOf(t.getFlags().asReadOnlyByteBuffer()))));
return new TagSet(new Project.NameKey(proto.getProjectName()), refs, tags);
}
void writeObject(ObjectOutputStream out) throws IOException {
out.writeInt(refs.size());
for (Map.Entry<String, CachedRef> e : refs.entrySet()) {
out.writeUTF(e.getKey());
out.writeInt(e.getValue().flag);
writeWithoutMarker(out, e.getValue().get());
}
out.writeInt(tags.size());
for (Tag tag : tags) {
writeWithoutMarker(out, tag);
out.writeObject(tag.refFlags);
}
TagSetProto toProto() {
ObjectIdConverter idConverter = ObjectIdConverter.create();
TagSetProto.Builder b = TagSetProto.newBuilder().setProjectName(projectName.get());
refs.forEach(
(n, cr) ->
b.putRef(
n,
CachedRefProto.newBuilder()
.setId(idConverter.toByteString(cr.get()))
.setFlag(cr.flag)
.build()));
tags.forEach(
t ->
b.addTag(
TagProto.newBuilder()
.setId(idConverter.toByteString(t))
.setFlags(ByteString.copyFrom(t.refFlags.toByteArray()))
.build()));
return b.build();
}
private boolean refresh(TagSet old, TagMatcher m) {
@@ -341,8 +374,17 @@ class TagSet {
return ref.getName().startsWith(Constants.R_TAGS);
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("projectName", projectName)
.add("refs", refs)
.add("tags", tags)
.toString();
}
static final class Tag extends ObjectIdOwnerMap.Entry {
private final BitSet refFlags;
@VisibleForTesting final BitSet refFlags;
Tag(AnyObjectId id, BitSet flags) {
super(id);
@@ -352,11 +394,15 @@ class TagSet {
boolean has(BitSet mask) {
return refFlags.intersects(mask);
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this).addValue(name()).add("refFlags", refFlags).toString();
}
}
private static final class CachedRef extends AtomicReference<ObjectId> {
private static final long serialVersionUID = 1L;
@VisibleForTesting
static final class CachedRef extends AtomicReference<ObjectId> {
final int flag;
CachedRef(Ref ref, int flag) {
@@ -367,6 +413,15 @@ class TagSet {
this.flag = flag;
set(id);
}
@Override
public String toString() {
ObjectId id = get();
return MoreObjects.toStringHelper(this)
.addValue(id != null ? id.name() : "null")
.add("flag", flag)
.toString();
}
}
private static final class TagWalk extends RevWalk {

33
java/com/google/gerrit/server/git/TagSetHolder.java
View File

@@ -16,7 +16,11 @@ package com.google.gerrit.server.git;
import static java.util.stream.Collectors.toList;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.cache.CacheSerializer;
import com.google.gerrit.server.cache.ProtoCacheSerializers;
import com.google.gerrit.server.cache.proto.Cache.TagSetHolderProto;
import java.util.Collection;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
@@ -24,7 +28,8 @@ import org.eclipse.jgit.lib.Repository;
public class TagSetHolder {
private final Object buildLock = new Object();
private final Project.NameKey projectName;
private volatile TagSet tags;
@Nullable private volatile TagSet tags;
TagSetHolder(Project.NameKey projectName) {
this.projectName = projectName;
@@ -94,4 +99,30 @@ public class TagSetHolder {
return cur;
}
}
enum Serializer implements CacheSerializer<TagSetHolder> {
INSTANCE;
@Override
public byte[] serialize(TagSetHolder object) {
TagSetHolderProto.Builder b =
TagSetHolderProto.newBuilder().setProjectName(object.projectName.get());
TagSet tags = object.tags;
if (tags != null) {
b.setTags(tags.toProto());
}
return ProtoCacheSerializers.toByteArray(b.build());
}
@Override
public TagSetHolder deserialize(byte[] in) {
TagSetHolderProto proto =
ProtoCacheSerializers.parseUnchecked(TagSetHolderProto.parser(), in);
TagSetHolder holder = new TagSetHolder(new Project.NameKey(proto.getProjectName()));
if (proto.hasTags()) {
holder.tags = TagSet.fromProto(proto.getTags());
}
return holder;
}
}
}