Merge changes I8c01f550,Ic53dfcf7,I20118b57
* changes: SubscriptionGraph: Make the factory an interface SubmoduleOp: Create CircularPathFinder class SubscriptionGraph: Container for subscription relation
This commit is contained in:
Alice Kober-Sotzek
41
java/com/google/gerrit/server/submit/CircularPathFinder.java
Normal file
41
java/com/google/gerrit/server/submit/CircularPathFinder.java
Normal file
@@ -0,0 +1,41 @@
|
||||
// Copyright (C) 2020 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.server.submit;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
|
||||
class CircularPathFinder {
|
||||
private CircularPathFinder() {}
|
||||
|
||||
/**
|
||||
* Prints a circular path according to the nodes in {@code p} and the start node {@code target}.
|
||||
*/
|
||||
public static <T> String printCircularPath(Collection<T> p, T target) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(target);
|
||||
ArrayList<T> reverseP = new ArrayList<>(p);
|
||||
Collections.reverse(reverseP);
|
||||
for (T t : reverseP) {
|
||||
sb.append("->");
|
||||
sb.append(t);
|
||||
if (t.equals(target)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
@@ -14,19 +14,14 @@
|
||||
|
||||
package com.google.gerrit.server.submit;
|
||||
|
||||
import static com.google.gerrit.server.project.ProjectCache.illegalState;
|
||||
import static java.util.Comparator.comparing;
|
||||
import static java.util.stream.Collectors.toList;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.MultimapBuilder;
|
||||
import com.google.common.collect.SetMultimap;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.gerrit.common.UsedAt;
|
||||
import com.google.gerrit.common.data.SubscribeSection;
|
||||
import com.google.gerrit.entities.BranchNameKey;
|
||||
import com.google.gerrit.entities.Project;
|
||||
import com.google.gerrit.entities.RefNames;
|
||||
import com.google.gerrit.entities.SubmoduleSubscription;
|
||||
import com.google.gerrit.exceptions.StorageException;
|
||||
import com.google.gerrit.extensions.restapi.RestApiException;
|
||||
@@ -46,17 +41,11 @@ import com.google.inject.Inject;
|
||||
import com.google.inject.Provider;
|
||||
import com.google.inject.Singleton;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Deque;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
@@ -72,10 +61,8 @@ import org.eclipse.jgit.lib.Config;
|
||||
import org.eclipse.jgit.lib.FileMode;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.PersonIdent;
|
||||
import org.eclipse.jgit.lib.Ref;
|
||||
import org.eclipse.jgit.revwalk.RevCommit;
|
||||
import org.eclipse.jgit.revwalk.RevWalk;
|
||||
import org.eclipse.jgit.transport.RefSpec;
|
||||
|
||||
public class SubmoduleOp {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
@@ -126,40 +113,15 @@ public class SubmoduleOp {
|
||||
}
|
||||
}
|
||||
|
||||
private final GitModules.Factory gitmodulesFactory;
|
||||
private final PersonIdent myIdent;
|
||||
private final ProjectCache projectCache;
|
||||
private final VerboseSuperprojectUpdate verboseSuperProject;
|
||||
private final boolean enableSuperProjectSubscriptions;
|
||||
private final long maxCombinedCommitMessageSize;
|
||||
private final long maxCommitMessages;
|
||||
private final MergeOpRepoManager orm;
|
||||
private final Map<BranchNameKey, GitModules> branchGitModules;
|
||||
|
||||
/** Branches updated as part of the enclosing submit or push batch. */
|
||||
private final ImmutableSet<BranchNameKey> updatedBranches;
|
||||
|
||||
/**
|
||||
* Branches in a superproject that contain submodule subscriptions, plus branches in submodules
|
||||
* which are subscribed to by some superproject.
|
||||
*/
|
||||
private final Set<BranchNameKey> affectedBranches;
|
||||
|
||||
/** Copy of {@link #affectedBranches}, sorted by submodule traversal order. */
|
||||
private final ImmutableSet<BranchNameKey> sortedBranches;
|
||||
|
||||
/** Multimap of superproject branch to submodule subscriptions contained in that branch. */
|
||||
private final SetMultimap<BranchNameKey, SubmoduleSubscription> targets;
|
||||
private final SubscriptionGraph.Factory subscriptionGraphFactory;
|
||||
private final SubscriptionGraph subscriptionGraph;
|
||||
|
||||
private final BranchTips branchTips = new BranchTips();
|
||||
/**
|
||||
* Multimap of superproject name to all branch names within that superproject which have submodule
|
||||
* subscriptions.
|
||||
*/
|
||||
private final SetMultimap<Project.NameKey, BranchNameKey> branchesByProject;
|
||||
|
||||
/** All branches subscribed by other projects. */
|
||||
private final Set<BranchNameKey> subscribedBranches;
|
||||
|
||||
private SubmoduleOp(
|
||||
GitModules.Factory gitmodulesFactory,
|
||||
@@ -169,239 +131,27 @@ public class SubmoduleOp {
|
||||
Set<BranchNameKey> updatedBranches,
|
||||
MergeOpRepoManager orm)
|
||||
throws SubmoduleConflictException {
|
||||
this.gitmodulesFactory = gitmodulesFactory;
|
||||
this.myIdent = myIdent;
|
||||
this.projectCache = projectCache;
|
||||
this.verboseSuperProject =
|
||||
cfg.getEnum("submodule", null, "verboseSuperprojectUpdate", VerboseSuperprojectUpdate.TRUE);
|
||||
this.enableSuperProjectSubscriptions =
|
||||
cfg.getBoolean("submodule", "enableSuperProjectSubscriptions", true);
|
||||
this.maxCombinedCommitMessageSize =
|
||||
cfg.getLong("submodule", "maxCombinedCommitMessageSize", 256 << 10);
|
||||
this.maxCommitMessages = cfg.getLong("submodule", "maxCommitMessages", 1000);
|
||||
this.orm = orm;
|
||||
this.updatedBranches = ImmutableSet.copyOf(updatedBranches);
|
||||
this.targets = MultimapBuilder.hashKeys().hashSetValues().build();
|
||||
this.affectedBranches = new HashSet<>();
|
||||
this.branchGitModules = new HashMap<>();
|
||||
this.branchesByProject = MultimapBuilder.hashKeys().hashSetValues().build();
|
||||
this.subscribedBranches = new HashSet<>();
|
||||
this.sortedBranches = calculateSubscriptionMaps();
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the internal maps used by the operation.
|
||||
*
|
||||
* <p>In addition to the return value, the following fields are populated as a side effect:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link #affectedBranches}
|
||||
* <li>{@link #targets}
|
||||
* <li>{@link #branchesByProject}
|
||||
* <li>{@link #subscribedBranches}
|
||||
* </ul>
|
||||
*
|
||||
* @return the ordered set to be stored in {@link #sortedBranches}.
|
||||
*/
|
||||
// TODO(dborowitz): This setup process is hard to follow, in large part due to the accumulation of
|
||||
// mutable maps, which makes this whole class difficult to understand.
|
||||
//
|
||||
// A cleaner architecture for this process might be:
|
||||
// 1. Separate out the code to parse submodule subscriptions and build up an in-memory data
|
||||
// structure representing the subscription graph, using a separate class with a properly-
|
||||
// documented interface.
|
||||
// 2. Walk the graph to produce a work plan. This would be a list of items indicating: create a
|
||||
// commit in project X reading branch tips for submodules S1..Sn and updating gitlinks in X.
|
||||
// 3. Execute the work plan, i.e. convert the items into BatchUpdate.Ops and add them to the
|
||||
// relevant updates.
|
||||
//
|
||||
// In addition to improving readability, this approach has the advantage of making (1) and (2)
|
||||
// testable using small tests.
|
||||
private ImmutableSet<BranchNameKey> calculateSubscriptionMaps()
|
||||
throws SubmoduleConflictException {
|
||||
if (!enableSuperProjectSubscriptions) {
|
||||
this.subscriptionGraphFactory =
|
||||
new SubscriptionGraph.DefaultFactory(gitmodulesFactory, projectCache, orm);
|
||||
if (cfg.getBoolean("submodule", "enableSuperProjectSubscriptions", true)) {
|
||||
this.subscriptionGraph = subscriptionGraphFactory.compute(updatedBranches);
|
||||
} else {
|
||||
logger.atFine().log("Updating superprojects disabled");
|
||||
return null;
|
||||
this.subscriptionGraph =
|
||||
SubscriptionGraph.createEmptyGraph(ImmutableSet.copyOf(updatedBranches));
|
||||
}
|
||||
|
||||
logger.atFine().log("Calculating superprojects - submodules map");
|
||||
LinkedHashSet<BranchNameKey> allVisited = new LinkedHashSet<>();
|
||||
for (BranchNameKey updatedBranch : updatedBranches) {
|
||||
if (allVisited.contains(updatedBranch)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
searchForSuperprojects(updatedBranch, new LinkedHashSet<>(), allVisited);
|
||||
}
|
||||
|
||||
// Since the searchForSuperprojects will add all branches (related or
|
||||
// unrelated) and ensure the superproject's branches get added first before
|
||||
// a submodule branch. Need remove all unrelated branches and reverse
|
||||
// the order.
|
||||
allVisited.retainAll(affectedBranches);
|
||||
reverse(allVisited);
|
||||
return ImmutableSet.copyOf(allVisited);
|
||||
}
|
||||
|
||||
private void searchForSuperprojects(
|
||||
BranchNameKey current,
|
||||
LinkedHashSet<BranchNameKey> currentVisited,
|
||||
LinkedHashSet<BranchNameKey> allVisited)
|
||||
throws SubmoduleConflictException {
|
||||
logger.atFine().log("Now processing %s", current);
|
||||
|
||||
if (currentVisited.contains(current)) {
|
||||
throw new SubmoduleConflictException(
|
||||
"Branch level circular subscriptions detected: "
|
||||
+ printCircularPath(currentVisited, current));
|
||||
}
|
||||
|
||||
if (allVisited.contains(current)) {
|
||||
return;
|
||||
}
|
||||
|
||||
currentVisited.add(current);
|
||||
try {
|
||||
Collection<SubmoduleSubscription> subscriptions =
|
||||
superProjectSubscriptionsForSubmoduleBranch(current);
|
||||
for (SubmoduleSubscription sub : subscriptions) {
|
||||
BranchNameKey superBranch = sub.getSuperProject();
|
||||
searchForSuperprojects(superBranch, currentVisited, allVisited);
|
||||
targets.put(superBranch, sub);
|
||||
branchesByProject.put(superBranch.project(), superBranch);
|
||||
affectedBranches.add(superBranch);
|
||||
affectedBranches.add(sub.getSubmodule());
|
||||
subscribedBranches.add(sub.getSubmodule());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new StorageException("Cannot find superprojects for " + current, e);
|
||||
}
|
||||
currentVisited.remove(current);
|
||||
allVisited.add(current);
|
||||
}
|
||||
|
||||
private static <T> void reverse(LinkedHashSet<T> set) {
|
||||
if (set == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Deque<T> q = new ArrayDeque<>(set);
|
||||
set.clear();
|
||||
|
||||
while (!q.isEmpty()) {
|
||||
set.add(q.removeLast());
|
||||
}
|
||||
}
|
||||
|
||||
private <T> String printCircularPath(LinkedHashSet<T> p, T target) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(target);
|
||||
ArrayList<T> reverseP = new ArrayList<>(p);
|
||||
Collections.reverse(reverseP);
|
||||
for (T t : reverseP) {
|
||||
sb.append("->");
|
||||
sb.append(t);
|
||||
if (t.equals(target)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private Collection<BranchNameKey> getDestinationBranches(BranchNameKey src, SubscribeSection s)
|
||||
throws IOException {
|
||||
Collection<BranchNameKey> ret = new HashSet<>();
|
||||
logger.atFine().log("Inspecting SubscribeSection %s", s);
|
||||
for (RefSpec r : s.getMatchingRefSpecs()) {
|
||||
logger.atFine().log("Inspecting [matching] ref %s", r);
|
||||
if (!r.matchSource(src.branch())) {
|
||||
continue;
|
||||
}
|
||||
if (r.isWildcard()) {
|
||||
// refs/heads/*[:refs/somewhere/*]
|
||||
ret.add(
|
||||
BranchNameKey.create(
|
||||
s.getProject(), r.expandFromSource(src.branch()).getDestination()));
|
||||
} else {
|
||||
// e.g. refs/heads/master[:refs/heads/stable]
|
||||
String dest = r.getDestination();
|
||||
if (dest == null) {
|
||||
dest = r.getSource();
|
||||
}
|
||||
ret.add(BranchNameKey.create(s.getProject(), dest));
|
||||
}
|
||||
}
|
||||
|
||||
for (RefSpec r : s.getMultiMatchRefSpecs()) {
|
||||
logger.atFine().log("Inspecting [all] ref %s", r);
|
||||
if (!r.matchSource(src.branch())) {
|
||||
continue;
|
||||
}
|
||||
OpenRepo or;
|
||||
try {
|
||||
or = orm.getRepo(s.getProject());
|
||||
} catch (NoSuchProjectException e) {
|
||||
// A project listed a non existent project to be allowed
|
||||
// to subscribe to it. Allow this for now, i.e. no exception is
|
||||
// thrown.
|
||||
continue;
|
||||
}
|
||||
|
||||
for (Ref ref : or.repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_HEADS)) {
|
||||
if (r.getDestination() != null && !r.matchDestination(ref.getName())) {
|
||||
continue;
|
||||
}
|
||||
BranchNameKey b = BranchNameKey.create(s.getProject(), ref.getName());
|
||||
if (!ret.contains(b)) {
|
||||
ret.add(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.atFine().log("Returning possible branches: %s for project %s", ret, s.getProject());
|
||||
return ret;
|
||||
}
|
||||
|
||||
private Collection<SubmoduleSubscription> superProjectSubscriptionsForSubmoduleBranch(
|
||||
BranchNameKey srcBranch) throws IOException {
|
||||
logger.atFine().log("Calculating possible superprojects for %s", srcBranch);
|
||||
Collection<SubmoduleSubscription> ret = new ArrayList<>();
|
||||
Project.NameKey srcProject = srcBranch.project();
|
||||
for (SubscribeSection s :
|
||||
projectCache
|
||||
.get(srcProject)
|
||||
.orElseThrow(illegalState(srcProject))
|
||||
.getSubscribeSections(srcBranch)) {
|
||||
logger.atFine().log("Checking subscribe section %s", s);
|
||||
Collection<BranchNameKey> branches = getDestinationBranches(srcBranch, s);
|
||||
for (BranchNameKey targetBranch : branches) {
|
||||
Project.NameKey targetProject = targetBranch.project();
|
||||
try {
|
||||
OpenRepo or = orm.getRepo(targetProject);
|
||||
ObjectId id = or.repo.resolve(targetBranch.branch());
|
||||
if (id == null) {
|
||||
logger.atFine().log("The branch %s doesn't exist.", targetBranch);
|
||||
continue;
|
||||
}
|
||||
} catch (NoSuchProjectException e) {
|
||||
logger.atFine().log("The project %s doesn't exist", targetProject);
|
||||
continue;
|
||||
}
|
||||
|
||||
GitModules m = branchGitModules.get(targetBranch);
|
||||
if (m == null) {
|
||||
m = gitmodulesFactory.create(targetBranch, orm);
|
||||
branchGitModules.put(targetBranch, m);
|
||||
}
|
||||
ret.addAll(m.subscribedTo(srcBranch));
|
||||
}
|
||||
}
|
||||
logger.atFine().log("Calculated superprojects for %s are %s", srcBranch, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@UsedAt(UsedAt.Project.PLUGIN_DELETE_PROJECT)
|
||||
public boolean hasSuperproject(BranchNameKey branch) {
|
||||
return subscribedBranches.contains(branch);
|
||||
return subscriptionGraph.hasSuperproject(branch);
|
||||
}
|
||||
|
||||
public void updateSuperProjects() throws RestApiException {
|
||||
@@ -414,11 +164,11 @@ public class SubmoduleOp {
|
||||
try {
|
||||
for (Project.NameKey project : projects) {
|
||||
// only need superprojects
|
||||
if (branchesByProject.containsKey(project)) {
|
||||
if (subscriptionGraph.isAffectedSuperProject(project)) {
|
||||
superProjects.add(project);
|
||||
// get a new BatchUpdate for the super project
|
||||
OpenRepo or = orm.getRepo(project);
|
||||
for (BranchNameKey branch : branchesByProject.get(project)) {
|
||||
for (BranchNameKey branch : subscriptionGraph.getAffectedSuperBranches(project)) {
|
||||
addOp(or.getUpdate(), branch);
|
||||
}
|
||||
}
|
||||
@@ -454,7 +204,7 @@ public class SubmoduleOp {
|
||||
int count = 0;
|
||||
|
||||
List<SubmoduleSubscription> subscriptions =
|
||||
targets.get(subscriber).stream()
|
||||
subscriptionGraph.getSubscriptions(subscriber).stream()
|
||||
.sorted(comparing(SubmoduleSubscription::getPath))
|
||||
.collect(toList());
|
||||
for (SubmoduleSubscription s : subscriptions) {
|
||||
@@ -507,7 +257,7 @@ public class SubmoduleOp {
|
||||
StringBuilder msgbuf = new StringBuilder();
|
||||
DirCache dc = readTree(or.rw, currentCommit);
|
||||
DirCacheEditor ed = dc.editor();
|
||||
for (SubmoduleSubscription s : targets.get(subscriber)) {
|
||||
for (SubmoduleSubscription s : subscriptionGraph.getSubscriptions(subscriber)) {
|
||||
updateSubmodule(dc, ed, msgbuf, s);
|
||||
}
|
||||
ed.finish();
|
||||
@@ -584,6 +334,7 @@ public class SubmoduleOp {
|
||||
}
|
||||
|
||||
CodeReviewCommit newCommit = maybeNewCommit.get();
|
||||
|
||||
if (Objects.equals(newCommit, oldCommit)) {
|
||||
// gitlink have already been updated for this submodule
|
||||
return null;
|
||||
@@ -669,11 +420,11 @@ public class SubmoduleOp {
|
||||
|
||||
ImmutableSet<Project.NameKey> getProjectsInOrder() throws SubmoduleConflictException {
|
||||
LinkedHashSet<Project.NameKey> projects = new LinkedHashSet<>();
|
||||
for (Project.NameKey project : branchesByProject.keySet()) {
|
||||
for (Project.NameKey project : subscriptionGraph.getAffectedSuperProjects()) {
|
||||
addAllSubmoduleProjects(project, new LinkedHashSet<>(), projects);
|
||||
}
|
||||
|
||||
for (BranchNameKey branch : updatedBranches) {
|
||||
for (BranchNameKey branch : subscriptionGraph.getUpdatedBranches()) {
|
||||
projects.add(branch.project());
|
||||
}
|
||||
return ImmutableSet.copyOf(projects);
|
||||
@@ -686,7 +437,8 @@ public class SubmoduleOp {
|
||||
throws SubmoduleConflictException {
|
||||
if (current.contains(project)) {
|
||||
throw new SubmoduleConflictException(
|
||||
"Project level circular subscriptions detected: " + printCircularPath(current, project));
|
||||
"Project level circular subscriptions detected: "
|
||||
+ CircularPathFinder.printCircularPath(current, project));
|
||||
}
|
||||
|
||||
if (projects.contains(project)) {
|
||||
@@ -695,8 +447,8 @@ public class SubmoduleOp {
|
||||
|
||||
current.add(project);
|
||||
Set<Project.NameKey> subprojects = new HashSet<>();
|
||||
for (BranchNameKey branch : branchesByProject.get(project)) {
|
||||
Collection<SubmoduleSubscription> subscriptions = targets.get(branch);
|
||||
for (BranchNameKey branch : subscriptionGraph.getAffectedSuperBranches(project)) {
|
||||
Collection<SubmoduleSubscription> subscriptions = subscriptionGraph.getSubscriptions(branch);
|
||||
for (SubmoduleSubscription s : subscriptions) {
|
||||
subprojects.add(s.getSubmodule().project());
|
||||
}
|
||||
@@ -712,15 +464,13 @@ public class SubmoduleOp {
|
||||
|
||||
ImmutableSet<BranchNameKey> getBranchesInOrder() {
|
||||
LinkedHashSet<BranchNameKey> branches = new LinkedHashSet<>();
|
||||
if (sortedBranches != null) {
|
||||
branches.addAll(sortedBranches);
|
||||
}
|
||||
branches.addAll(updatedBranches);
|
||||
branches.addAll(subscriptionGraph.getSortedSuperprojectAndSubmoduleBranches());
|
||||
branches.addAll(subscriptionGraph.getUpdatedBranches());
|
||||
return ImmutableSet.copyOf(branches);
|
||||
}
|
||||
|
||||
boolean hasSubscription(BranchNameKey branch) {
|
||||
return targets.containsKey(branch);
|
||||
return subscriptionGraph.hasSubscription(branch);
|
||||
}
|
||||
|
||||
void addBranchTip(BranchNameKey branch, CodeReviewCommit tip) {
|
||||
|
||||
375
java/com/google/gerrit/server/submit/SubscriptionGraph.java
Normal file
375
java/com/google/gerrit/server/submit/SubscriptionGraph.java
Normal file
@@ -0,0 +1,375 @@
|
||||
// Copyright (C) 2020 The Android Open Source Project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package com.google.gerrit.server.submit;
|
||||
|
||||
import static com.google.gerrit.server.project.ProjectCache.illegalState;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.collect.MultimapBuilder;
|
||||
import com.google.common.collect.SetMultimap;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.gerrit.common.data.SubscribeSection;
|
||||
import com.google.gerrit.entities.BranchNameKey;
|
||||
import com.google.gerrit.entities.Project;
|
||||
import com.google.gerrit.entities.RefNames;
|
||||
import com.google.gerrit.entities.SubmoduleSubscription;
|
||||
import com.google.gerrit.exceptions.StorageException;
|
||||
import com.google.gerrit.server.project.NoSuchProjectException;
|
||||
import com.google.gerrit.server.project.ProjectCache;
|
||||
import com.google.gerrit.server.submit.MergeOpRepoManager.OpenRepo;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Deque;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.Ref;
|
||||
import org.eclipse.jgit.transport.RefSpec;
|
||||
|
||||
/**
|
||||
* A container which stores subscription relationship. A SubscriptionGraph is calculated every time
|
||||
* changes are pushed. Some branches are updated in these changes, and if these branches are
|
||||
* subscribed by other projects, SubscriptionGraph would record information about these updated
|
||||
* branches and branches/projects affected.
|
||||
*/
|
||||
public class SubscriptionGraph {
|
||||
/** Branches updated as part of the enclosing submit or push batch. */
|
||||
private final ImmutableSet<BranchNameKey> updatedBranches;
|
||||
|
||||
/**
|
||||
* All branches affected, including those in superprojects and submodules, sorted by submodule
|
||||
* traversal order. To support nested subscriptions, GitLink commits need to be updated in order.
|
||||
* The closer to topological "leaf", the earlier a commit should be updated.
|
||||
*
|
||||
* <p>For example, there are three projects, top level project p1 subscribed to p2, p2 subscribed
|
||||
* to bottom level project p3. When submit a change for p3. We need update both p2 and p1. To be
|
||||
* more precise, we need update p2 first and then update p1.
|
||||
*/
|
||||
private final ImmutableSet<BranchNameKey> sortedBranches;
|
||||
|
||||
/** Multimap of superproject branch to submodule subscriptions contained in that branch. */
|
||||
private final ImmutableSetMultimap<BranchNameKey, SubmoduleSubscription> targets;
|
||||
|
||||
/**
|
||||
* Multimap of superproject name to all branch names within that superproject which have submodule
|
||||
* subscriptions.
|
||||
*/
|
||||
private final ImmutableSetMultimap<Project.NameKey, BranchNameKey> branchesByProject;
|
||||
|
||||
/** All branches subscribed by other projects. */
|
||||
private final ImmutableSet<BranchNameKey> subscribedBranches;
|
||||
|
||||
public SubscriptionGraph(
|
||||
Set<BranchNameKey> updatedBranches,
|
||||
SetMultimap<BranchNameKey, SubmoduleSubscription> targets,
|
||||
SetMultimap<Project.NameKey, BranchNameKey> branchesByProject,
|
||||
Set<BranchNameKey> subscribedBranches,
|
||||
Set<BranchNameKey> sortedBranches) {
|
||||
this.updatedBranches = ImmutableSet.copyOf(updatedBranches);
|
||||
this.targets = ImmutableSetMultimap.copyOf(targets);
|
||||
this.branchesByProject = ImmutableSetMultimap.copyOf(branchesByProject);
|
||||
this.subscribedBranches = ImmutableSet.copyOf(subscribedBranches);
|
||||
this.sortedBranches = ImmutableSet.copyOf(sortedBranches);
|
||||
}
|
||||
|
||||
/** Returns an empty {@code SubscriptionGraph}. */
|
||||
static SubscriptionGraph createEmptyGraph(Set<BranchNameKey> updatedBranches) {
|
||||
return new SubscriptionGraph(
|
||||
updatedBranches,
|
||||
ImmutableSetMultimap.of(),
|
||||
ImmutableSetMultimap.of(),
|
||||
ImmutableSet.of(),
|
||||
ImmutableSet.of());
|
||||
}
|
||||
|
||||
/** Get branches updated as part of the enclosing submit or push batch. */
|
||||
ImmutableSet<BranchNameKey> getUpdatedBranches() {
|
||||
return updatedBranches;
|
||||
}
|
||||
|
||||
/** Get all superprojects affected. */
|
||||
ImmutableSet<Project.NameKey> getAffectedSuperProjects() {
|
||||
return branchesByProject.keySet();
|
||||
}
|
||||
|
||||
/** See if a {@code project} is a superproject affected. */
|
||||
boolean isAffectedSuperProject(Project.NameKey project) {
|
||||
return branchesByProject.containsKey(project);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all branches within the superproject {@code project} which have submodule
|
||||
* subscriptions.
|
||||
*/
|
||||
ImmutableSet<BranchNameKey> getAffectedSuperBranches(Project.NameKey project) {
|
||||
return branchesByProject.get(project);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all affected branches, including the submodule branches and superproject branches, sorted
|
||||
* by traversal order.
|
||||
*
|
||||
* @see SubscriptionGraph#sortedBranches
|
||||
*/
|
||||
ImmutableSet<BranchNameKey> getSortedSuperprojectAndSubmoduleBranches() {
|
||||
return sortedBranches;
|
||||
}
|
||||
|
||||
/** Check if a {@code branch} is a submodule of a superproject. */
|
||||
boolean hasSuperproject(BranchNameKey branch) {
|
||||
return subscribedBranches.contains(branch);
|
||||
}
|
||||
|
||||
/** See if a {@code branch} is a superproject branch affected. */
|
||||
boolean hasSubscription(BranchNameKey branch) {
|
||||
return targets.containsKey(branch);
|
||||
}
|
||||
|
||||
/** Get all related {@code SubmoduleSubscription}s whose super branch is {@code branch}. */
|
||||
ImmutableSet<SubmoduleSubscription> getSubscriptions(BranchNameKey branch) {
|
||||
return targets.get(branch);
|
||||
}
|
||||
|
||||
public interface Factory {
|
||||
SubscriptionGraph compute(Set<BranchNameKey> updatedBranches) throws SubmoduleConflictException;
|
||||
}
|
||||
|
||||
static class DefaultFactory implements Factory {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private final ProjectCache projectCache;
|
||||
private final GitModules.Factory gitmodulesFactory;
|
||||
private final Map<BranchNameKey, GitModules> branchGitModules;
|
||||
private final MergeOpRepoManager orm;
|
||||
|
||||
// Fields required to the constructor of SubscriptionGraph.
|
||||
/** All affected branches, including those in superprojects and submodules. */
|
||||
private final Set<BranchNameKey> affectedBranches;
|
||||
|
||||
/** @see SubscriptionGraph#targets */
|
||||
private final SetMultimap<BranchNameKey, SubmoduleSubscription> targets;
|
||||
|
||||
/** @see SubscriptionGraph#branchesByProject */
|
||||
private final SetMultimap<Project.NameKey, BranchNameKey> branchesByProject;
|
||||
|
||||
/** @see SubscriptionGraph#subscribedBranches */
|
||||
private final Set<BranchNameKey> subscribedBranches;
|
||||
|
||||
DefaultFactory(
|
||||
GitModules.Factory gitmodulesFactory, ProjectCache projectCache, MergeOpRepoManager orm) {
|
||||
this.gitmodulesFactory = gitmodulesFactory;
|
||||
this.projectCache = projectCache;
|
||||
this.orm = orm;
|
||||
this.branchGitModules = new HashMap<>();
|
||||
|
||||
this.affectedBranches = new HashSet<>();
|
||||
this.targets = MultimapBuilder.hashKeys().hashSetValues().build();
|
||||
this.branchesByProject = MultimapBuilder.hashKeys().hashSetValues().build();
|
||||
this.subscribedBranches = new HashSet<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public SubscriptionGraph compute(Set<BranchNameKey> updatedBranches)
|
||||
throws SubmoduleConflictException {
|
||||
return new SubscriptionGraph(
|
||||
updatedBranches,
|
||||
targets,
|
||||
branchesByProject,
|
||||
subscribedBranches,
|
||||
calculateSubscriptionMaps(updatedBranches));
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the internal maps used by the operation.
|
||||
*
|
||||
* <p>In addition to the return value, the following fields are populated as a side effect:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link #affectedBranches}
|
||||
* <li>{@link #targets}
|
||||
* <li>{@link #branchesByProject}
|
||||
* <li>{@link #subscribedBranches}
|
||||
* </ul>
|
||||
*
|
||||
* @return the ordered set to be stored in {@link #sortedBranches}.
|
||||
*/
|
||||
private Set<BranchNameKey> calculateSubscriptionMaps(Set<BranchNameKey> updatedBranches)
|
||||
throws SubmoduleConflictException {
|
||||
logger.atFine().log("Calculating superprojects - submodules map");
|
||||
LinkedHashSet<BranchNameKey> allVisited = new LinkedHashSet<>();
|
||||
for (BranchNameKey updatedBranch : updatedBranches) {
|
||||
if (allVisited.contains(updatedBranch)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
searchForSuperprojects(updatedBranch, new LinkedHashSet<>(), allVisited);
|
||||
}
|
||||
|
||||
// Since the searchForSuperprojects will add all branches (related or
|
||||
// unrelated) and ensure the superproject's branches get added first before
|
||||
// a submodule branch. Need remove all unrelated branches and reverse
|
||||
// the order.
|
||||
allVisited.retainAll(affectedBranches);
|
||||
reverse(allVisited);
|
||||
return allVisited;
|
||||
}
|
||||
|
||||
private void searchForSuperprojects(
|
||||
BranchNameKey current,
|
||||
LinkedHashSet<BranchNameKey> currentVisited,
|
||||
LinkedHashSet<BranchNameKey> allVisited)
|
||||
throws SubmoduleConflictException {
|
||||
logger.atFine().log("Now processing %s", current);
|
||||
|
||||
if (currentVisited.contains(current)) {
|
||||
throw new SubmoduleConflictException(
|
||||
"Branch level circular subscriptions detected: "
|
||||
+ CircularPathFinder.printCircularPath(currentVisited, current));
|
||||
}
|
||||
|
||||
if (allVisited.contains(current)) {
|
||||
return;
|
||||
}
|
||||
|
||||
currentVisited.add(current);
|
||||
try {
|
||||
Collection<SubmoduleSubscription> subscriptions =
|
||||
superProjectSubscriptionsForSubmoduleBranch(current);
|
||||
for (SubmoduleSubscription sub : subscriptions) {
|
||||
BranchNameKey superBranch = sub.getSuperProject();
|
||||
searchForSuperprojects(superBranch, currentVisited, allVisited);
|
||||
targets.put(superBranch, sub);
|
||||
branchesByProject.put(superBranch.project(), superBranch);
|
||||
affectedBranches.add(superBranch);
|
||||
affectedBranches.add(sub.getSubmodule());
|
||||
subscribedBranches.add(sub.getSubmodule());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new StorageException("Cannot find superprojects for " + current, e);
|
||||
}
|
||||
currentVisited.remove(current);
|
||||
allVisited.add(current);
|
||||
}
|
||||
|
||||
private Collection<BranchNameKey> getDestinationBranches(BranchNameKey src, SubscribeSection s)
|
||||
throws IOException {
|
||||
Collection<BranchNameKey> ret = new HashSet<>();
|
||||
logger.atFine().log("Inspecting SubscribeSection %s", s);
|
||||
for (RefSpec r : s.getMatchingRefSpecs()) {
|
||||
logger.atFine().log("Inspecting [matching] ref %s", r);
|
||||
if (!r.matchSource(src.branch())) {
|
||||
continue;
|
||||
}
|
||||
if (r.isWildcard()) {
|
||||
// refs/heads/*[:refs/somewhere/*]
|
||||
ret.add(
|
||||
BranchNameKey.create(
|
||||
s.getProject(), r.expandFromSource(src.branch()).getDestination()));
|
||||
} else {
|
||||
// e.g. refs/heads/master[:refs/heads/stable]
|
||||
String dest = r.getDestination();
|
||||
if (dest == null) {
|
||||
dest = r.getSource();
|
||||
}
|
||||
ret.add(BranchNameKey.create(s.getProject(), dest));
|
||||
}
|
||||
}
|
||||
|
||||
for (RefSpec r : s.getMultiMatchRefSpecs()) {
|
||||
logger.atFine().log("Inspecting [all] ref %s", r);
|
||||
if (!r.matchSource(src.branch())) {
|
||||
continue;
|
||||
}
|
||||
OpenRepo or;
|
||||
try {
|
||||
or = orm.getRepo(s.getProject());
|
||||
} catch (NoSuchProjectException e) {
|
||||
// A project listed a non existent project to be allowed
|
||||
// to subscribe to it. Allow this for now, i.e. no exception is
|
||||
// thrown.
|
||||
continue;
|
||||
}
|
||||
|
||||
for (Ref ref : or.repo.getRefDatabase().getRefsByPrefix(RefNames.REFS_HEADS)) {
|
||||
if (r.getDestination() != null && !r.matchDestination(ref.getName())) {
|
||||
continue;
|
||||
}
|
||||
BranchNameKey b = BranchNameKey.create(s.getProject(), ref.getName());
|
||||
if (!ret.contains(b)) {
|
||||
ret.add(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.atFine().log("Returning possible branches: %s for project %s", ret, s.getProject());
|
||||
return ret;
|
||||
}
|
||||
|
||||
private Collection<SubmoduleSubscription> superProjectSubscriptionsForSubmoduleBranch(
|
||||
BranchNameKey srcBranch) throws IOException {
|
||||
logger.atFine().log("Calculating possible superprojects for %s", srcBranch);
|
||||
Collection<SubmoduleSubscription> ret = new ArrayList<>();
|
||||
Project.NameKey srcProject = srcBranch.project();
|
||||
for (SubscribeSection s :
|
||||
projectCache
|
||||
.get(srcProject)
|
||||
.orElseThrow(illegalState(srcProject))
|
||||
.getSubscribeSections(srcBranch)) {
|
||||
logger.atFine().log("Checking subscribe section %s", s);
|
||||
Collection<BranchNameKey> branches = getDestinationBranches(srcBranch, s);
|
||||
for (BranchNameKey targetBranch : branches) {
|
||||
Project.NameKey targetProject = targetBranch.project();
|
||||
try {
|
||||
OpenRepo or = orm.getRepo(targetProject);
|
||||
ObjectId id = or.repo.resolve(targetBranch.branch());
|
||||
if (id == null) {
|
||||
logger.atFine().log("The branch %s doesn't exist.", targetBranch);
|
||||
continue;
|
||||
}
|
||||
} catch (NoSuchProjectException e) {
|
||||
logger.atFine().log("The project %s doesn't exist", targetProject);
|
||||
continue;
|
||||
}
|
||||
|
||||
GitModules m = branchGitModules.get(targetBranch);
|
||||
if (m == null) {
|
||||
m = gitmodulesFactory.create(targetBranch, orm);
|
||||
branchGitModules.put(targetBranch, m);
|
||||
}
|
||||
ret.addAll(m.subscribedTo(srcBranch));
|
||||
}
|
||||
}
|
||||
logger.atFine().log("Calculated superprojects for %s are %s", srcBranch, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
private static <T> void reverse(LinkedHashSet<T> set) {
|
||||
if (set == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Deque<T> q = new ArrayDeque<>(set);
|
||||
set.clear();
|
||||
|
||||
while (!q.isEmpty()) {
|
||||
set.add(q.removeLast());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user