opendev/gerrit
Code Issues Proposed changes

Rework SearchingChangeCacheImpl

Browse Source 
When filtering refs, we previously had two mechanisms for getting change
information somewhat efficiently. If the instance is running as a
master, we used SearchingChangeCacheImpl to get all change information
from the change index. We parsed this and used if for permission
filtering.

In slaves, we scanned the repo and parsed the changes from disk. There
are two reasons for this: (1) we don't have a change index
available in slaves. (2) There was no cross-machine eviction.

In the master, SearchingChangeCacheImpl was disabled by default, so
without manual configuration, we issued one index call for each
list-refs call.

In addition, we accepted potential staleness of the index. If a change
was moved (= the target branch changed) or marked as private and we
missed the change index update, it would still be available in Git.

This commit reworks the way how we retrieve and store change data for
ref filtering:

We remove SearchingChangeCacheImpl and replace it with ChangeRefCache.
ChangeRefCache has a different caching mechanism. The key contains
project, changeId and the SHA1 of the meta ref. This makes it so that we
can spare any custom eviction logic and have the Guava cache do it's
internal eviction purely based on cache size.

This makes the cache suitable for Gerrit slaves as well.

In addition, we adapt the way how we load change information: If the
change index is available, we bootstrap the cache once per JVM for each
project using the index. All subsequent updates are done incrementally
using the persisted ChangeNotesCache. This drastically cuts down on the
number of index calls we do (one per project per instance vs one per
request) while not sacrificing on the benefits. Due to the (comparably)
low number of change updates per instance and the (comparably) high number
of list ref calls, we will do a very small amount of incremental updates
of the cache for any given list ref call.

For slaves that don't have a change index available, we don't do any
bootstrapping for now and will use the ChangeNotesCache as requests come
in. In case this is too slow, we can easily bootstrap the cache in a
slave using a lifecycle listener. This will be added in the future if
there is a need.

Why do we need this cache at all now that we have a ChangeNotesCache?
To efficiently filter change refs, all (or nearly all) of the
information needs to be in-memory. The ChangeNotesCache is rather large
and on googlesource.com we can't hold all of it in memory.
ChangeRefCache has a small enough footprint that we can. This might
very well be true for other Gerrit instances as well. In case it is not,
administrators can choose to disable ChangeRefCache and have their ref
filtering be backed by ChangeNotesCache.

With this commit, we add integration tests for the new cache and remove
complexity from DefaultRefFilter.

Change-Id: I5eda9d411e97925e3e8b450fe32693a936164f96
This commit is contained in:
Patrick Hiesel
2019-01-09 13:45:52 +01:00
parent 572f4e3804
commit f5c89c1a6f
10 changed files with 576 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
java/com/google/gerrit/server/permissions/DefaultRefFilter.java
View File

@@ -14,7 +14,6 @@
package com.google.gerrit.server.permissions;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.gerrit.reviewdb.client.RefNames.REFS_CACHE_AUTOMERGE;
import static com.google.gerrit.reviewdb.client.RefNames.REFS_CHANGES;
import static com.google.gerrit.reviewdb.client.RefNames.REFS_CONFIG;
@@ -22,9 +21,7 @@ import static com.google.gerrit.reviewdb.client.RefNames.REFS_USERS_SELF;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.extensions.restapi.AuthException;
@@ -41,12 +38,11 @@ import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.account.GroupCache;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.SearchingChangeCacheImpl;
import com.google.gerrit.server.git.ChangeRefCache;
import com.google.gerrit.server.git.TagCache;
import com.google.gerrit.server.git.TagMatcher;
import com.google.gerrit.server.group.InternalGroup;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.notedb.ChangeNotes.Factory.ChangeNotesResult;
import com.google.gerrit.server.permissions.PermissionBackend.RefFilterOptions;
import com.google.gerrit.server.project.ProjectState;
import com.google.gerrit.server.query.change.ChangeData;
@@ -55,10 +51,11 @@ import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.Ref;
@@ -74,7 +71,7 @@ class DefaultRefFilter {
private final TagCache tagCache;
private final ChangeNotes.Factory changeNotesFactory;
@Nullable private final SearchingChangeCacheImpl changeCache;
@Nullable private final ChangeRefCache changeCache;
private final GroupCache groupCache;
private final PermissionBackend permissionBackend;
private final ProjectControl projectControl;
@@ -84,14 +81,14 @@ class DefaultRefFilter {
private final Counter0 fullFilterCount;
private final Counter0 skipFilterCount;
private final boolean skipFullRefEvaluationIfAllRefsAreVisible;
private Map<Change.Id, Branch.NameKey> visibleChanges;
private final Map<Change.Id, Branch.NameKey> visibleChanges;
private final Set<Change.Id> inVisibleChanges;
@Inject
DefaultRefFilter(
TagCache tagCache,
ChangeNotes.Factory changeNotesFactory,
@Nullable SearchingChangeCacheImpl changeCache,
@Nullable ChangeRefCache changeCache,
GroupCache groupCache,
PermissionBackend permissionBackend,
@GerritServerConfig Config config,
@@ -121,6 +118,9 @@ class DefaultRefFilter {
"Rate of ref filter operations where we skip full evaluation"
+ " because the user can read all refs")
.setRate());
// TODO(hiesel): Rework who can see change edits so that we can keep just a single set here.
this.visibleChanges = new HashMap<>();
this.inVisibleChanges = new HashSet<>();
}
Map<String, Ref> filter(Map<String, Ref> refs, Repository repo, RefFilterOptions opts)
@@ -161,6 +161,7 @@ class DefaultRefFilter {
Map<String, Ref> result = new HashMap<>();
List<Ref> deferredTags = new ArrayList<>();
changeCache.bootstrapIfNecessary(projectState.getNameKey());
for (Ref ref : refs.values()) {
String name = ref.getName();
@@ -299,110 +300,64 @@ class DefaultRefFilter {
}
private boolean visible(Repository repo, Change.Id changeId) throws PermissionBackendException {
if (visibleChanges == null) {
if (changeCache == null) {
visibleChanges = visibleChangesByScan(repo);
} else {
visibleChanges = visibleChangesBySearch();
}
if (visibleChanges.containsKey(changeId)) {
return true;
}
if (inVisibleChanges.contains(changeId)) {
return false;
}
// TODO(hiesel): The project state should be checked once in the beginning an left alone
// thereafter.
if (!projectState.statePermitsRead()) {
return false;
}
Project.NameKey project = projectState.getNameKey();
try {
ChangeData cd =
changeCache.getChangeData(
project, changeId, repo.exactRef(RefNames.changeMetaRef(changeId)).getObjectId());
ChangeNotes notes = changeNotesFactory.createFromIndexedChange(cd.change());
try {
permissionBackendForProject.indexedChange(cd, notes).check(ChangePermission.READ);
visibleChanges.put(cd.getId(), cd.change().getDest());
return true;
} catch (AuthException e) {
inVisibleChanges.add(changeId);
return false;
}
} catch (OrmException | IOException e) {
throw new PermissionBackendException(
"Cannot load change " + changeId + " for project " + project + ", assuming not visible",
e);
}
return visibleChanges.containsKey(changeId);
}
private boolean visibleEdit(Repository repo, String name) throws PermissionBackendException {
Change.Id id = Change.Id.fromEditRefPart(name);
// Initialize if it wasn't yet
if (visibleChanges == null) {
visible(repo, id);
}
if (id == null) {
return false;
}
if (!visible(repo, id)) {
// Can't see the change, so can't see the edit.
return false;
}
if (user.isIdentifiedUser()
&& name.startsWith(RefNames.refsEditPrefix(user.asIdentifiedUser().getAccountId()))
&& visible(repo, id)) {
&& name.startsWith(RefNames.refsEditPrefix(user.asIdentifiedUser().getAccountId()))) {
// Own edit
return true;
}
if (visibleChanges.containsKey(id)) {
try {
// Default to READ_PRIVATE_CHANGES as there is no special permission for reading edits.
permissionBackendForProject
.ref(visibleChanges.get(id).get())
.check(RefPermission.READ_PRIVATE_CHANGES);
return true;
} catch (AuthException e) {
return false;
}
}
return false;
}
private Map<Change.Id, Branch.NameKey> visibleChangesBySearch()
throws PermissionBackendException {
Project.NameKey project = projectState.getNameKey();
try {
Map<Change.Id, Branch.NameKey> visibleChanges = new HashMap<>();
for (ChangeData cd : changeCache.getChangeData(project)) {
ChangeNotes notes = changeNotesFactory.createFromIndexedChange(cd.change());
if (!projectState.statePermitsRead()) {
continue;
}
try {
permissionBackendForProject.indexedChange(cd, notes).check(ChangePermission.READ);
visibleChanges.put(cd.getId(), cd.change().getDest());
} catch (AuthException e) {
// Do nothing.
}
}
return visibleChanges;
} catch (OrmException e) {
logger.atSevere().withCause(e).log(
"Cannot load changes for project %s, assuming no changes are visible", project);
return Collections.emptyMap();
}
}
private Map<Change.Id, Branch.NameKey> visibleChangesByScan(Repository repo)
throws PermissionBackendException {
Project.NameKey p = projectState.getNameKey();
ImmutableList<ChangeNotesResult> changes;
try {
changes = changeNotesFactory.scan(repo, p).collect(toImmutableList());
} catch (IOException e) {
logger.atSevere().withCause(e).log(
"Cannot load changes for project %s, assuming no changes are visible", p);
return Collections.emptyMap();
}
Map<Change.Id, Branch.NameKey> result = Maps.newHashMapWithExpectedSize(changes.size());
for (ChangeNotesResult notesResult : changes) {
ChangeNotes notes = toNotes(notesResult);
if (notes != null) {
result.put(notes.getChangeId(), notes.getChange().getDest());
}
}
return result;
}
@Nullable
private ChangeNotes toNotes(ChangeNotesResult r) throws PermissionBackendException {
if (r.error().isPresent()) {
logger.atWarning().withCause(r.error().get()).log(
"Failed to load change %s in %s", r.id(), projectState.getName());
return null;
}
if (!projectState.statePermitsRead()) {
return null;
}
try {
permissionBackendForProject.change(r.notes()).check(ChangePermission.READ);
return r.notes();
// Default to READ_PRIVATE_CHANGES as there is no special permission for reading edits.
permissionBackendForProject
.ref(visibleChanges.get(id).get())
.check(RefPermission.READ_PRIVATE_CHANGES);
return true;
} catch (AuthException e) {
// Skip.
return false;
}
return null;
}
private boolean isMetadata(String name) {