opendev/gerrit
Code Issues Proposed changes

Merge changes I8c01f550,Ic53dfcf7,I20118b57

Browse Source 
* changes:
  SubscriptionGraph: Make the factory an interface
  SubmoduleOp: Create CircularPathFinder class
  SubscriptionGraph: Container for subscription relation
This commit is contained in:
Alice Kober-Sotzek
2020-06-19 11:16:03 +00:00
committed by Gerrit Code Review
parent 10f2f9d7d6 cffd4719b0
commit a40c28ff1d
4 changed files with 654 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
java/com/google/gerrit/server/submit/CircularPathFinder.java Normal file
View File

@@ -0,0 +1,41 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.submit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
class CircularPathFinder {
private CircularPathFinder() {}
/**
* Prints a circular path according to the nodes in {@code p} and the start node {@code target}.
*/
public static <T> String printCircularPath(Collection<T> p, T target) {
StringBuilder sb = new StringBuilder();
sb.append(target);
ArrayList<T> reverseP = new ArrayList<>(p);
Collections.reverse(reverseP);
for (T t : reverseP) {
sb.append("->");
sb.append(t);
if (t.equals(target)) {
break;
}
}
return sb.toString();
}
}

298
java/com/google/gerrit/server/submit/SubmoduleOp.java
View File

@@ -14,19 +14,14 @@
package com.google.gerrit.server.submit;
import static com.google.gerrit.server.project.ProjectCache.illegalState;
import static java.util.Comparator.comparing;
import static java.util.stream.Collectors.toList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.SetMultimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.UsedAt;
import com.google.gerrit.common.data.SubscribeSection;
import com.google.gerrit.entities.BranchNameKey;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.entities.SubmoduleSubscription;
import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.extensions.restapi.RestApiException;
@@ -46,17 +41,11 @@ import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
@@ -72,10 +61,8 @@ import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.RefSpec;
public class SubmoduleOp {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -126,40 +113,15 @@ public class SubmoduleOp {
}
}
private final GitModules.Factory gitmodulesFactory;
private final PersonIdent myIdent;
private final ProjectCache projectCache;
private final VerboseSuperprojectUpdate verboseSuperProject;
private final boolean enableSuperProjectSubscriptions;
private final long maxCombinedCommitMessageSize;
private final long maxCommitMessages;
private final MergeOpRepoManager orm;
private final Map<BranchNameKey, GitModules> branchGitModules;
/** Branches updated as part of the enclosing submit or push batch. */
private final ImmutableSet<BranchNameKey> updatedBranches;
/**
* Branches in a superproject that contain submodule subscriptions, plus branches in submodules
* which are subscribed to by some superproject.
*/
private final Set<BranchNameKey> affectedBranches;
/** Copy of {@link #affectedBranches}, sorted by submodule traversal order. */
private final ImmutableSet<BranchNameKey> sortedBranches;
/** Multimap of superproject branch to submodule subscriptions contained in that branch. */
private final SetMultimap<BranchNameKey, SubmoduleSubscription> targets;
private final SubscriptionGraph.Factory subscriptionGraphFactory;
private final SubscriptionGraph subscriptionGraph;
private final BranchTips branchTips = new BranchTips();
/**
* Multimap of superproject name to all branch names within that superproject which have submodule
* subscriptions.
*/
private final SetMultimap<Project.NameKey, BranchNameKey> branchesByProject;
/** All branches subscribed by other projects. */
private final Set<BranchNameKey> subscribedBranches;
private SubmoduleOp(
GitModules.Factory gitmodulesFactory,
@@ -169,239 +131,27 @@ public class SubmoduleOp {
Set<BranchNameKey> updatedBranches,
MergeOpRepoManager orm)
throws SubmoduleConflictException {
this.gitmodulesFactory = gitmodulesFactory;
this.myIdent = myIdent;
this.projectCache = projectCache;
this.verboseSuperProject =
cfg.getEnum("submodule", null, "verboseSuperprojectUpdate", VerboseSuperprojectUpdate.TRUE);
this.enableSuperProjectSubscriptions =
cfg.getBoolean("submodule", "enableSuperProjectSubscriptions", true);
this.maxCombinedCommitMessageSize =
cfg.getLong("submodule", "maxCombinedCommitMessageSize", 256 << 10);
this.maxCommitMessages = cfg.getLong("submodule", "maxCommitMessages", 1000);
this.orm = orm;
this.updatedBranches = ImmutableSet.copyOf(updatedBranches);
this.targets = MultimapBuilder.hashKeys().hashSetValues().build();
this.affectedBranches = new HashSet<>();
this.branchGitModules = new HashMap<>();
this.branchesByProject = MultimapBuilder.hashKeys().hashSetValues().build();
this.subscribedBranches = new HashSet<>();
this.sortedBranches = calculateSubscriptionMaps();
}
/**
* Calculate the internal maps used by the operation.
*
* <p>In addition to the return value, the following fields are populated as a side effect:
*
* <ul>
* <li>{@link #affectedBranches}
* <li>{@link #targets}
* <li>{@link #branchesByProject}
* <li>{@link #subscribedBranches}
* </ul>
*
* @return the ordered set to be stored in {@link #sortedBranches}.
*/
// TODO(dborowitz): This setup process is hard to follow, in large part due to the accumulation of
// mutable maps, which makes this whole class difficult to understand.
//
// A cleaner architecture for this process might be:
// 1. Separate out the code to parse submodule subscriptions and build up an in-memory data
// structure representing the subscription graph, using a separate class with a properly-
// documented interface.
// 2. Walk the graph to produce a work plan. This would be a list of items indicating: create a
// commit in project X reading branch tips for submodules S1..Sn and updating gitlinks in X.
// 3. Execute the work plan, i.e. convert the items into BatchUpdate.Ops and add them to the
// relevant updates.
//
// In addition to improving readability, this approach has the advantage of making (1) and (2)
// testable using small tests.
private ImmutableSet<BranchNameKey> calculateSubscriptionMaps()
throws SubmoduleConflictException {
if (!enableSuperProjectSubscriptions) {
this.subscriptionGraphFactory =
new SubscriptionGraph.DefaultFactory(gitmodulesFactory, projectCache, orm);
if (cfg.getBoolean("submodule", "enableSuperProjectSubscriptions", true)) {
this.subscriptionGraph = subscriptionGraphFactory.compute(updatedBranches);
} else {
logger.atFine().log("Updating superprojects disabled");
return null;
this.subscriptionGraph =
SubscriptionGraph.createEmptyGraph(ImmutableSet.copyOf(updatedBranches));
}
logger.atFine().log("Calculating superprojects - submodules map");
LinkedHashSet<BranchNameKey> allVisited = new LinkedHashSet<>();
for (BranchNameKey updatedBranch : updatedBranches) {
if (allVisited.contains(updatedBranch)) {
continue;
}
searchForSuperprojects(updatedBranch, new LinkedHashSet<>(), allVisited);
}
// Since the searchForSuperprojects will add all branches (related or
// unrelated) and ensure the superproject's branches get added first before
// a submodule branch. Need remove all unrelated branches and reverse
// the order.
allVisited.retainAll(affectedBranches);
reverse(allVisited);
return ImmutableSet.copyOf(allVisited);
}
private void searchForSuperprojects(
BranchNameKey current,
LinkedHashSet<BranchNameKey> currentVisited,
LinkedHashSet<BranchNameKey> allVisited)
throws SubmoduleConflictException {
logger.atFine().log("Now processing %s", current);
if (currentVisited.contains(current)) {
throw new SubmoduleConflictException(
"Branch level circular subscriptions detected: "
+ printCircularPath(currentVisited, current));
}
if (allVisited.contains(current)) {
return;
}
currentVisited.add(current);
try {
Collection<SubmoduleSubscription> subscriptions =
superProjectSubscriptionsForSubmoduleBranch(current);
for (SubmoduleSubscription sub : subscriptions) {
BranchNameKey superBranch = sub.getSuperProject();
searchForSuperprojects(superBranch, currentVisited, allVisited);
targets.put(superBranch, sub);
branchesByProject.put(superBranch.project(), superBranch);
affectedBranches.add(superBranch);
affectedBranches.add(sub.getSubmodule());
subscribedBranches.add(sub.getSubmodule());
}
} catch (IOException e) {
throw new StorageException("Cannot find superprojects for " + current, e);
}
currentVisited.remove(current);
allVisited.add(current);
}
private static <T> void reverse(LinkedHashSet<T> set) {
if (set == null) {
return;
}
Deque<T> q = new ArrayDeque<>(set);
set.clear();
while (!q.isEmpty()) {
set.add(q.removeLast());
}
}
private <T> String printCircularPath(LinkedHashSet<T> p, T target) {
StringBuilder sb = new StringBuilder();
sb.append(target);
ArrayList<T> reverseP = new ArrayList<>(p);
Collections.reverse(reverseP);
for (T t : reverseP) {
sb.append("->");
sb.append(t);
if (t.equals(target)) {
break;
}
}
return sb.toString();
}
private Collection<BranchNameKey> getDestinationBranches(BranchNameKey src, SubscribeSection s)
throws IOException {
Collection<BranchNameKey> ret = new HashSet<>();
logger.atFine().log("Inspecting SubscribeSection %s", s);
for (RefSpec r : s.getMatchingRefSpecs()) {
logger.atFine().log("Inspecting [matching] ref %s", r);
if (!r.matchSource(src.branch())) {
continue;
}
if (r.isWildcard()) {
// refs/heads/*[:refs/somewhere/*]
ret.add(
BranchNameKey.create(
s.getProject(), r.expandFromSource(src.branch()).getDestination()));
} else {
// e.g. refs/heads/master[:refs/heads/stable]
String dest = r.getDestination();
if (dest == null) {
dest = r.getSource();
}
ret.add(BranchNameKey.create(s.getProject(), dest));
}
}
for (RefSpec r : s.getMultiMatchRefSpecs()) {
logger.atFine().log("Inspecting [all] ref %s", r);
if (!r.matchSource(src.branch())) {
continue;
}
OpenRepo or;
try {
or = orm.getRepo(s.getProject());
} catch (NoSuchProjectException e) {
// A project listed a non existent project to be allowed
// to subscribe to it. Allow this for now, i.e. no exception is
// thrown.
continue;
}
for (Ref ref : or.repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_HEADS)) {
if (r.getDestination() != null && !r.matchDestination(ref.getName())) {
continue;
}
BranchNameKey b = BranchNameKey.create(s.getProject(), ref.getName());
if (!ret.contains(b)) {
ret.add(b);
}
}
}
logger.atFine().log("Returning possible branches: %s for project %s", ret, s.getProject());
return ret;
}
private Collection<SubmoduleSubscription> superProjectSubscriptionsForSubmoduleBranch(
BranchNameKey srcBranch) throws IOException {
logger.atFine().log("Calculating possible superprojects for %s", srcBranch);
Collection<SubmoduleSubscription> ret = new ArrayList<>();
Project.NameKey srcProject = srcBranch.project();
for (SubscribeSection s :
projectCache
.get(srcProject)
.orElseThrow(illegalState(srcProject))
.getSubscribeSections(srcBranch)) {
logger.atFine().log("Checking subscribe section %s", s);
Collection<BranchNameKey> branches = getDestinationBranches(srcBranch, s);
for (BranchNameKey targetBranch : branches) {
Project.NameKey targetProject = targetBranch.project();
try {
OpenRepo or = orm.getRepo(targetProject);
ObjectId id = or.repo.resolve(targetBranch.branch());
if (id == null) {
logger.atFine().log("The branch %s doesn't exist.", targetBranch);
continue;
}
} catch (NoSuchProjectException e) {
logger.atFine().log("The project %s doesn't exist", targetProject);
continue;
}
GitModules m = branchGitModules.get(targetBranch);
if (m == null) {
m = gitmodulesFactory.create(targetBranch, orm);
branchGitModules.put(targetBranch, m);
}
ret.addAll(m.subscribedTo(srcBranch));
}
}
logger.atFine().log("Calculated superprojects for %s are %s", srcBranch, ret);
return ret;
}
@UsedAt(UsedAt.Project.PLUGIN_DELETE_PROJECT)
public boolean hasSuperproject(BranchNameKey branch) {
return subscribedBranches.contains(branch);
return subscriptionGraph.hasSuperproject(branch);
}
public void updateSuperProjects() throws RestApiException {
@@ -414,11 +164,11 @@ public class SubmoduleOp {
try {
for (Project.NameKey project : projects) {
// only need superprojects
if (branchesByProject.containsKey(project)) {
if (subscriptionGraph.isAffectedSuperProject(project)) {
superProjects.add(project);
// get a new BatchUpdate for the super project
OpenRepo or = orm.getRepo(project);
for (BranchNameKey branch : branchesByProject.get(project)) {
for (BranchNameKey branch : subscriptionGraph.getAffectedSuperBranches(project)) {
addOp(or.getUpdate(), branch);
}
}
@@ -454,7 +204,7 @@ public class SubmoduleOp {
int count = 0;
List<SubmoduleSubscription> subscriptions =
targets.get(subscriber).stream()
subscriptionGraph.getSubscriptions(subscriber).stream()
.sorted(comparing(SubmoduleSubscription::getPath))
.collect(toList());
for (SubmoduleSubscription s : subscriptions) {
@@ -507,7 +257,7 @@ public class SubmoduleOp {
StringBuilder msgbuf = new StringBuilder();
DirCache dc = readTree(or.rw, currentCommit);
DirCacheEditor ed = dc.editor();
for (SubmoduleSubscription s : targets.get(subscriber)) {
for (SubmoduleSubscription s : subscriptionGraph.getSubscriptions(subscriber)) {
updateSubmodule(dc, ed, msgbuf, s);
}
ed.finish();
@@ -584,6 +334,7 @@ public class SubmoduleOp {
}
CodeReviewCommit newCommit = maybeNewCommit.get();
if (Objects.equals(newCommit, oldCommit)) {
// gitlink have already been updated for this submodule
return null;
@@ -669,11 +420,11 @@ public class SubmoduleOp {
ImmutableSet<Project.NameKey> getProjectsInOrder() throws SubmoduleConflictException {
LinkedHashSet<Project.NameKey> projects = new LinkedHashSet<>();
for (Project.NameKey project : branchesByProject.keySet()) {
for (Project.NameKey project : subscriptionGraph.getAffectedSuperProjects()) {
addAllSubmoduleProjects(project, new LinkedHashSet<>(), projects);
}
for (BranchNameKey branch : updatedBranches) {
for (BranchNameKey branch : subscriptionGraph.getUpdatedBranches()) {
projects.add(branch.project());
}
return ImmutableSet.copyOf(projects);
@@ -686,7 +437,8 @@ public class SubmoduleOp {
throws SubmoduleConflictException {
if (current.contains(project)) {
throw new SubmoduleConflictException(
"Project level circular subscriptions detected: " + printCircularPath(current, project));
"Project level circular subscriptions detected: "
+ CircularPathFinder.printCircularPath(current, project));
}
if (projects.contains(project)) {
@@ -695,8 +447,8 @@ public class SubmoduleOp {
current.add(project);
Set<Project.NameKey> subprojects = new HashSet<>();
for (BranchNameKey branch : branchesByProject.get(project)) {
Collection<SubmoduleSubscription> subscriptions = targets.get(branch);
for (BranchNameKey branch : subscriptionGraph.getAffectedSuperBranches(project)) {
Collection<SubmoduleSubscription> subscriptions = subscriptionGraph.getSubscriptions(branch);
for (SubmoduleSubscription s : subscriptions) {
subprojects.add(s.getSubmodule().project());
}
@@ -712,15 +464,13 @@ public class SubmoduleOp {
ImmutableSet<BranchNameKey> getBranchesInOrder() {
LinkedHashSet<BranchNameKey> branches = new LinkedHashSet<>();
if (sortedBranches != null) {
branches.addAll(sortedBranches);
}
branches.addAll(updatedBranches);
branches.addAll(subscriptionGraph.getSortedSuperprojectAndSubmoduleBranches());
branches.addAll(subscriptionGraph.getUpdatedBranches());
return ImmutableSet.copyOf(branches);
}
boolean hasSubscription(BranchNameKey branch) {
return targets.containsKey(branch);
return subscriptionGraph.hasSubscription(branch);
}
void addBranchTip(BranchNameKey branch, CodeReviewCommit tip) {

375
java/com/google/gerrit/server/submit/SubscriptionGraph.java Normal file
View File

@@ -0,0 +1,375 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.submit;
import static com.google.gerrit.server.project.ProjectCache.illegalState;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.SetMultimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.data.SubscribeSection;
import com.google.gerrit.entities.BranchNameKey;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.entities.SubmoduleSubscription;
import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.server.project.NoSuchProjectException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.submit.MergeOpRepoManager.OpenRepo;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.transport.RefSpec;
/**
* A container which stores subscription relationship. A SubscriptionGraph is calculated every time
* changes are pushed. Some branches are updated in these changes, and if these branches are
* subscribed by other projects, SubscriptionGraph would record information about these updated
* branches and branches/projects affected.
*/
public class SubscriptionGraph {
/** Branches updated as part of the enclosing submit or push batch. */
private final ImmutableSet<BranchNameKey> updatedBranches;
/**
* All branches affected, including those in superprojects and submodules, sorted by submodule
* traversal order. To support nested subscriptions, GitLink commits need to be updated in order.
* The closer to topological "leaf", the earlier a commit should be updated.
*
* <p>For example, there are three projects, top level project p1 subscribed to p2, p2 subscribed
* to bottom level project p3. When submit a change for p3. We need update both p2 and p1. To be
* more precise, we need update p2 first and then update p1.
*/
private final ImmutableSet<BranchNameKey> sortedBranches;
/** Multimap of superproject branch to submodule subscriptions contained in that branch. */
private final ImmutableSetMultimap<BranchNameKey, SubmoduleSubscription> targets;
/**
* Multimap of superproject name to all branch names within that superproject which have submodule
* subscriptions.
*/
private final ImmutableSetMultimap<Project.NameKey, BranchNameKey> branchesByProject;
/** All branches subscribed by other projects. */
private final ImmutableSet<BranchNameKey> subscribedBranches;
public SubscriptionGraph(
Set<BranchNameKey> updatedBranches,
SetMultimap<BranchNameKey, SubmoduleSubscription> targets,
SetMultimap<Project.NameKey, BranchNameKey> branchesByProject,
Set<BranchNameKey> subscribedBranches,
Set<BranchNameKey> sortedBranches) {
this.updatedBranches = ImmutableSet.copyOf(updatedBranches);
this.targets = ImmutableSetMultimap.copyOf(targets);
this.branchesByProject = ImmutableSetMultimap.copyOf(branchesByProject);
this.subscribedBranches = ImmutableSet.copyOf(subscribedBranches);
this.sortedBranches = ImmutableSet.copyOf(sortedBranches);
}
/** Returns an empty {@code SubscriptionGraph}. */
static SubscriptionGraph createEmptyGraph(Set<BranchNameKey> updatedBranches) {
return new SubscriptionGraph(
updatedBranches,
ImmutableSetMultimap.of(),
ImmutableSetMultimap.of(),
ImmutableSet.of(),
ImmutableSet.of());
}
/** Get branches updated as part of the enclosing submit or push batch. */
ImmutableSet<BranchNameKey> getUpdatedBranches() {
return updatedBranches;
}
/** Get all superprojects affected. */
ImmutableSet<Project.NameKey> getAffectedSuperProjects() {
return branchesByProject.keySet();
}
/** See if a {@code project} is a superproject affected. */
boolean isAffectedSuperProject(Project.NameKey project) {
return branchesByProject.containsKey(project);
}
/**
* Returns all branches within the superproject {@code project} which have submodule
* subscriptions.
*/
ImmutableSet<BranchNameKey> getAffectedSuperBranches(Project.NameKey project) {
return branchesByProject.get(project);
}
/**
* Get all affected branches, including the submodule branches and superproject branches, sorted
* by traversal order.
*
* @see SubscriptionGraph#sortedBranches
*/
ImmutableSet<BranchNameKey> getSortedSuperprojectAndSubmoduleBranches() {
return sortedBranches;
}
/** Check if a {@code branch} is a submodule of a superproject. */
boolean hasSuperproject(BranchNameKey branch) {
return subscribedBranches.contains(branch);
}
/** See if a {@code branch} is a superproject branch affected. */
boolean hasSubscription(BranchNameKey branch) {
return targets.containsKey(branch);
}
/** Get all related {@code SubmoduleSubscription}s whose super branch is {@code branch}. */
ImmutableSet<SubmoduleSubscription> getSubscriptions(BranchNameKey branch) {
return targets.get(branch);
}
public interface Factory {
SubscriptionGraph compute(Set<BranchNameKey> updatedBranches) throws SubmoduleConflictException;
}
static class DefaultFactory implements Factory {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final ProjectCache projectCache;
private final GitModules.Factory gitmodulesFactory;
private final Map<BranchNameKey, GitModules> branchGitModules;
private final MergeOpRepoManager orm;
// Fields required to the constructor of SubscriptionGraph.
/** All affected branches, including those in superprojects and submodules. */
private final Set<BranchNameKey> affectedBranches;
/** @see SubscriptionGraph#targets */
private final SetMultimap<BranchNameKey, SubmoduleSubscription> targets;
/** @see SubscriptionGraph#branchesByProject */
private final SetMultimap<Project.NameKey, BranchNameKey> branchesByProject;
/** @see SubscriptionGraph#subscribedBranches */
private final Set<BranchNameKey> subscribedBranches;
DefaultFactory(
GitModules.Factory gitmodulesFactory, ProjectCache projectCache, MergeOpRepoManager orm) {
this.gitmodulesFactory = gitmodulesFactory;
this.projectCache = projectCache;
this.orm = orm;
this.branchGitModules = new HashMap<>();
this.affectedBranches = new HashSet<>();
this.targets = MultimapBuilder.hashKeys().hashSetValues().build();
this.branchesByProject = MultimapBuilder.hashKeys().hashSetValues().build();
this.subscribedBranches = new HashSet<>();
}
@Override
public SubscriptionGraph compute(Set<BranchNameKey> updatedBranches)
throws SubmoduleConflictException {
return new SubscriptionGraph(
updatedBranches,
targets,
branchesByProject,
subscribedBranches,
calculateSubscriptionMaps(updatedBranches));
}
/**
* Calculate the internal maps used by the operation.
*
* <p>In addition to the return value, the following fields are populated as a side effect:
*
* <ul>
* <li>{@link #affectedBranches}
* <li>{@link #targets}
* <li>{@link #branchesByProject}
* <li>{@link #subscribedBranches}
* </ul>
*
* @return the ordered set to be stored in {@link #sortedBranches}.
*/
private Set<BranchNameKey> calculateSubscriptionMaps(Set<BranchNameKey> updatedBranches)
throws SubmoduleConflictException {
logger.atFine().log("Calculating superprojects - submodules map");
LinkedHashSet<BranchNameKey> allVisited = new LinkedHashSet<>();
for (BranchNameKey updatedBranch : updatedBranches) {
if (allVisited.contains(updatedBranch)) {
continue;
}
searchForSuperprojects(updatedBranch, new LinkedHashSet<>(), allVisited);
}
// Since the searchForSuperprojects will add all branches (related or
// unrelated) and ensure the superproject's branches get added first before
// a submodule branch. Need remove all unrelated branches and reverse
// the order.
allVisited.retainAll(affectedBranches);
reverse(allVisited);
return allVisited;
}
private void searchForSuperprojects(
BranchNameKey current,
LinkedHashSet<BranchNameKey> currentVisited,
LinkedHashSet<BranchNameKey> allVisited)
throws SubmoduleConflictException {
logger.atFine().log("Now processing %s", current);
if (currentVisited.contains(current)) {
throw new SubmoduleConflictException(
"Branch level circular subscriptions detected: "
+ CircularPathFinder.printCircularPath(currentVisited, current));
}
if (allVisited.contains(current)) {
return;
}
currentVisited.add(current);
try {
Collection<SubmoduleSubscription> subscriptions =
superProjectSubscriptionsForSubmoduleBranch(current);
for (SubmoduleSubscription sub : subscriptions) {
BranchNameKey superBranch = sub.getSuperProject();
searchForSuperprojects(superBranch, currentVisited, allVisited);
targets.put(superBranch, sub);
branchesByProject.put(superBranch.project(), superBranch);
affectedBranches.add(superBranch);
affectedBranches.add(sub.getSubmodule());
subscribedBranches.add(sub.getSubmodule());
}
} catch (IOException e) {
throw new StorageException("Cannot find superprojects for " + current, e);
}
currentVisited.remove(current);
allVisited.add(current);
}
private Collection<BranchNameKey> getDestinationBranches(BranchNameKey src, SubscribeSection s)
throws IOException {
Collection<BranchNameKey> ret = new HashSet<>();
logger.atFine().log("Inspecting SubscribeSection %s", s);
for (RefSpec r : s.getMatchingRefSpecs()) {
logger.atFine().log("Inspecting [matching] ref %s", r);
if (!r.matchSource(src.branch())) {
continue;
}
if (r.isWildcard()) {
// refs/heads/*[:refs/somewhere/*]
ret.add(
BranchNameKey.create(
s.getProject(), r.expandFromSource(src.branch()).getDestination()));
} else {
// e.g. refs/heads/master[:refs/heads/stable]
String dest = r.getDestination();
if (dest == null) {
dest = r.getSource();
}
ret.add(BranchNameKey.create(s.getProject(), dest));
}
}
for (RefSpec r : s.getMultiMatchRefSpecs()) {
logger.atFine().log("Inspecting [all] ref %s", r);
if (!r.matchSource(src.branch())) {
continue;
}
OpenRepo or;
try {
or = orm.getRepo(s.getProject());
} catch (NoSuchProjectException e) {
// A project listed a non existent project to be allowed
// to subscribe to it. Allow this for now, i.e. no exception is
// thrown.
continue;
}
for (Ref ref : or.repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_HEADS)) {
if (r.getDestination() != null && !r.matchDestination(ref.getName())) {
continue;
}
BranchNameKey b = BranchNameKey.create(s.getProject(), ref.getName());
if (!ret.contains(b)) {
ret.add(b);
}
}
}
logger.atFine().log("Returning possible branches: %s for project %s", ret, s.getProject());
return ret;
}
private Collection<SubmoduleSubscription> superProjectSubscriptionsForSubmoduleBranch(
BranchNameKey srcBranch) throws IOException {
logger.atFine().log("Calculating possible superprojects for %s", srcBranch);
Collection<SubmoduleSubscription> ret = new ArrayList<>();
Project.NameKey srcProject = srcBranch.project();
for (SubscribeSection s :
projectCache
.get(srcProject)
.orElseThrow(illegalState(srcProject))
.getSubscribeSections(srcBranch)) {
logger.atFine().log("Checking subscribe section %s", s);
Collection<BranchNameKey> branches = getDestinationBranches(srcBranch, s);
for (BranchNameKey targetBranch : branches) {
Project.NameKey targetProject = targetBranch.project();
try {
OpenRepo or = orm.getRepo(targetProject);
ObjectId id = or.repo.resolve(targetBranch.branch());
if (id == null) {
logger.atFine().log("The branch %s doesn't exist.", targetBranch);
continue;
}
} catch (NoSuchProjectException e) {
logger.atFine().log("The project %s doesn't exist", targetProject);
continue;
}
GitModules m = branchGitModules.get(targetBranch);
if (m == null) {
m = gitmodulesFactory.create(targetBranch, orm);
branchGitModules.put(targetBranch, m);
}
ret.addAll(m.subscribedTo(srcBranch));
}
}
logger.atFine().log("Calculated superprojects for %s are %s", srcBranch, ret);
return ret;
}
private static <T> void reverse(LinkedHashSet<T> set) {
if (set == null) {
return;
}
Deque<T> q = new ArrayDeque<>(set);
set.clear();
while (!q.isEmpty()) {
set.add(q.removeLast());
}
}
}
}

214
javatests/com/google/gerrit/server/submit/SubscriptionGraphTest.java Normal file
View File

@@ -0,0 +1,214 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.submit;
import static com.google.common.truth.Truth.assertThat;
import static com.google.gerrit.testing.GerritJUnit.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.gerrit.common.data.SubscribeSection;
import com.google.gerrit.entities.BranchNameKey;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.SubmoduleSubscription;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.gerrit.server.submit.SubscriptionGraph.DefaultFactory;
import com.google.gerrit.testing.InMemoryRepositoryManager;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
public class SubscriptionGraphTest {
private static final String TEST_PATH = "test/path";
private static final Project.NameKey SUPER_PROJECT = Project.nameKey("Superproject");
private static final Project.NameKey SUB_PROJECT = Project.nameKey("Subproject");
private static final BranchNameKey SUPER_BRANCH =
BranchNameKey.create(SUPER_PROJECT, "refs/heads/one");
private static final BranchNameKey SUB_BRANCH =
BranchNameKey.create(SUB_PROJECT, "refs/heads/one");
private InMemoryRepositoryManager repoManager = new InMemoryRepositoryManager();
private MergeOpRepoManager mergeOpRepoManager;
@Mock GitModules.Factory mockGitModulesFactory = mock(GitModules.Factory.class);
@Mock ProjectCache mockProjectCache = mock(ProjectCache.class);
@Mock ProjectState mockProjectState = mock(ProjectState.class);
@Before
public void setUp() throws Exception {
when(mockProjectCache.get(any())).thenReturn(Optional.of(mockProjectState));
mergeOpRepoManager = new MergeOpRepoManager(repoManager, mockProjectCache, null, null);
GitModules emptyMockGitModules = mock(GitModules.class);
when(emptyMockGitModules.subscribedTo(any())).thenReturn(ImmutableSet.of());
when(mockGitModulesFactory.create(any(), any())).thenReturn(emptyMockGitModules);
TestRepository<Repository> superProject = createRepo(SUPER_PROJECT);
TestRepository<Repository> submoduleProject = createRepo(SUB_PROJECT);
// Make sure that SUPER_BRANCH and SUB_BRANCH can be subscribed.
allowSubscription(SUPER_BRANCH);
allowSubscription(SUB_BRANCH);
setSubscription(SUB_BRANCH, ImmutableList.of(SUPER_BRANCH));
setSubscription(SUPER_BRANCH, ImmutableList.of());
createBranch(
superProject, SUPER_BRANCH, superProject.commit().message("Initial commit").create());
createBranch(
submoduleProject, SUB_BRANCH, submoduleProject.commit().message("Initial commit").create());
}
@Test
public void oneSuperprojectOneSubmodule() throws Exception {
SubscriptionGraph.Factory factory =
new DefaultFactory(mockGitModulesFactory, mockProjectCache, mergeOpRepoManager);
SubscriptionGraph subscriptionGraph = factory.compute(ImmutableSet.of(SUB_BRANCH));
assertThat(subscriptionGraph.getAffectedSuperProjects()).containsExactly(SUPER_PROJECT);
assertThat(subscriptionGraph.getAffectedSuperBranches(SUPER_PROJECT))
.containsExactly(SUPER_BRANCH);
assertThat(subscriptionGraph.getSubscriptions(SUPER_BRANCH))
.containsExactly(new SubmoduleSubscription(SUPER_BRANCH, SUB_BRANCH, TEST_PATH));
assertThat(subscriptionGraph.hasSuperproject(SUB_BRANCH)).isTrue();
assertThat(subscriptionGraph.getSortedSuperprojectAndSubmoduleBranches())
.containsExactly(SUB_BRANCH, SUPER_BRANCH)
.inOrder();
}
@Test
public void circularSubscription() throws Exception {
SubscriptionGraph.Factory factory =
new DefaultFactory(mockGitModulesFactory, mockProjectCache, mergeOpRepoManager);
setSubscription(SUPER_BRANCH, ImmutableList.of(SUB_BRANCH));
SubmoduleConflictException e =
assertThrows(
SubmoduleConflictException.class, () -> factory.compute(ImmutableSet.of(SUB_BRANCH)));
String expectedErrorMessage =
"Subproject,refs/heads/one->Superproject,refs/heads/one->Subproject,refs/heads/one";
assertThat(e).hasMessageThat().contains(expectedErrorMessage);
}
@Test
public void multipleSuperprojectsToMultipleSubmodules() throws Exception {
// Create superprojects and subprojects.
Project.NameKey superProject1 = Project.nameKey("superproject1");
Project.NameKey superProject2 = Project.nameKey("superproject2");
Project.NameKey subProject1 = Project.nameKey("subproject1");
Project.NameKey subProject2 = Project.nameKey("subproject2");
TestRepository<Repository> superProjectRepo1 = createRepo(superProject1);
TestRepository<Repository> superProjectRepo2 = createRepo(superProject2);
TestRepository<Repository> submoduleRepo1 = createRepo(subProject1);
TestRepository<Repository> submoduleRepo2 = createRepo(subProject2);
// Initialize super branches.
BranchNameKey superBranch1 = BranchNameKey.create(superProject1, "refs/heads/one");
BranchNameKey superBranch2 = BranchNameKey.create(superProject2, "refs/heads/one");
createBranch(
superProjectRepo1,
superBranch1,
superProjectRepo1.commit().message("Initial commit").create());
createBranch(
superProjectRepo2,
superBranch2,
superProjectRepo2.commit().message("Initial commit").create());
// Initialize sub branches.
BranchNameKey submoduleBranch1 = BranchNameKey.create(subProject1, "refs/heads/one");
BranchNameKey submoduleBranch2 = BranchNameKey.create(subProject1, "refs/heads/two");
BranchNameKey submoduleBranch3 = BranchNameKey.create(subProject2, "refs/heads/one");
createBranch(
submoduleRepo1, submoduleBranch1, submoduleRepo1.commit().message("Commit1").create());
createBranch(
submoduleRepo1, submoduleBranch2, submoduleRepo1.commit().message("Commit2").create());
createBranch(
submoduleRepo2, submoduleBranch3, submoduleRepo2.commit().message("Commit1").create());
allowSubscription(submoduleBranch1);
allowSubscription(submoduleBranch2);
allowSubscription(submoduleBranch3);
// Initialize subscriptions.
setSubscription(submoduleBranch1, ImmutableList.of(superBranch1, superBranch2));
setSubscription(submoduleBranch2, ImmutableList.of(superBranch1));
setSubscription(submoduleBranch3, ImmutableList.of(superBranch1, superBranch2));
SubscriptionGraph.Factory factory =
new DefaultFactory(mockGitModulesFactory, mockProjectCache, mergeOpRepoManager);
SubscriptionGraph subscriptionGraph =
factory.compute(ImmutableSet.of(submoduleBranch1, submoduleBranch2));
assertThat(subscriptionGraph.getAffectedSuperProjects())
.containsExactly(superProject1, superProject2);
assertThat(subscriptionGraph.getAffectedSuperBranches(superProject1))
.containsExactly(superBranch1);
assertThat(subscriptionGraph.getAffectedSuperBranches(superProject2))
.containsExactly(superBranch2);
assertThat(subscriptionGraph.getSubscriptions(superBranch1))
.containsExactly(
new SubmoduleSubscription(superBranch1, submoduleBranch1, TEST_PATH),
new SubmoduleSubscription(superBranch1, submoduleBranch2, TEST_PATH));
assertThat(subscriptionGraph.getSubscriptions(superBranch2))
.containsExactly(new SubmoduleSubscription(superBranch2, submoduleBranch1, TEST_PATH));
assertThat(subscriptionGraph.hasSuperproject(submoduleBranch1)).isTrue();
assertThat(subscriptionGraph.hasSuperproject(submoduleBranch2)).isTrue();
assertThat(subscriptionGraph.hasSuperproject(submoduleBranch3)).isFalse();
assertThat(subscriptionGraph.getSortedSuperprojectAndSubmoduleBranches())
.containsExactly(submoduleBranch2, submoduleBranch1, superBranch2, superBranch1)
.inOrder();
}
private TestRepository<Repository> createRepo(Project.NameKey project) throws Exception {
Repository repo = repoManager.createRepository(project);
return new TestRepository<>(repo);
}
private void createBranch(TestRepository<Repository> repo, BranchNameKey branch, RevCommit commit)
throws Exception {
repo.update(branch.branch(), commit);
}
private void allowSubscription(BranchNameKey branch) {
SubscribeSection s = new SubscribeSection(branch.project());
s.addMultiMatchRefSpec("refs/heads/*:refs/heads/*");
when(mockProjectState.getSubscribeSections(branch)).thenReturn(ImmutableSet.of(s));
}
private void setSubscription(
BranchNameKey submoduleBranch, List<BranchNameKey> superprojectBranches) {
List<SubmoduleSubscription> subscriptions =
superprojectBranches.stream()
.map(
(targetBranch) ->
new SubmoduleSubscription(targetBranch, submoduleBranch, TEST_PATH))
.collect(Collectors.toList());
GitModules mockGitModules = mock(GitModules.class);
when(mockGitModules.subscribedTo(submoduleBranch)).thenReturn(subscriptions);
when(mockGitModulesFactory.create(submoduleBranch, mergeOpRepoManager))
.thenReturn(mockGitModules);
}
}