diff --git a/Documentation/user-submodules.txt b/Documentation/user-submodules.txt index 8ba31e7597..a8a0262d6f 100644 --- a/Documentation/user-submodules.txt +++ b/Documentation/user-submodules.txt @@ -1,82 +1,118 @@ = Gerrit Code Review - Superproject subscription to submodules updates +[[automatic_update]] == Description - Gerrit supports a custom git superproject feature for tracking submodules. This feature is useful for automatic updates on superprojects whenever -a change is merged on tracked submodules. To take advantage of this -feature, one should add submodule(s) to a local working copy of a -superproject, edit the created .gitmodules configuration file to -have a branch field on each submodule section with the value of the -submodule branch it is subscribing to, commit the changes, push and -merge the commit. +a change is merged on tracked submodules. -When a commit is merged to a project, the commit content is scanned -to identify if it registers git submodules (if the commit registers -any gitlinks and .gitmodules file with required info) and if so, -a new submodule subscription is registered. +When a superproject is subscribed to a submodule, it is not +required to push/merge commits to this superproject to update the +gitlink to the submodule. Whenever a commit is merged in a submodule, +its subscribed superproject is updated by Gerrit. -When a new commit of a registered submodule is merged, Gerrit -automatically updates the subscribers to the submodule with a new -commit having the updated gitlinks. +Imagine a superproject called 'super' having a branch called 'dev' +having subscribed to a submodule 'sub' on a branch 'dev-of-sub'. When a commit +is merged in branch 'dev-of-sub' of 'sub' project, Gerrit automatically +creates a new commit on branch 'dev' of 'super' updating the gitlink +to point to the just merged commit. + +To take advantage of this feature, one should: + +. ensure superproject subscriptions are enabled on the server via + link:config-gerrit.html#submodule.enableSuperProjectSubscriptions[submodule.enableSuperProjectSubscriptions] +. configure the submodule to allow having a superproject subscribed +. ensure the .gitmodules file of the superproject includes +.. a branch field +.. a url that starts with the link:config-gerrit.html#gerrit.canonicalWebUrl[`gerrit.canonicalWebUrl`] + +When a commit in a project is merged, Gerrit checks for superprojects +that are subscribed to the the project and automatically updates those +superprojects with a commit that updates the gilink for the project. This feature is enabled by default and can be disabled via link:config-gerrit.html#submodule.enableSuperProjectSubscriptions[submodule.enableSuperProjectSubscriptions] in the server configuration. -== Git Submodules Overview +== Git submodules overview -Submodules are a git feature that allows an external repository to be +Submodules are a Git feature that allows an external repository to be attached inside a repository at a specific path. The objective here is to provide a brief overview, further details can be found -in the official git submodule command documentation. +in the official Git submodule documentation. -Imagine a repository called 'super' and another one called 'a'. -Also consider 'a' available in a running Gerrit instance on "server". -With this feature, one could attach 'a' inside of 'super' repository -at path 'a' by executing the following command when being inside +Imagine a repository called 'super' and another one called 'sub'. +Also consider 'sub' available in a running Gerrit instance on "server". +With this feature, one could attach 'sub' inside of 'super' repository +at path 'sub' by executing the following command when being inside 'super': ===== -git submodule add ssh://server/a a +git submodule add ssh://server/sub sub ===== Still considering the above example, after its execution notice that -inside the local repository 'super' the 'a' folder is considered a -gitlink to the external repository 'a'. Also notice a file called +inside the local repository 'super' the 'sub' folder is considered a +gitlink to the external repository 'sub'. Also notice a file called .gitmodules is created (it is a configuration file containing the -subscription of 'a'). To provide the SHA-1 each gitlink points to in +subscription of 'sub'). To provide the SHA-1 each gitlink points to in the external repository, one should use the command: ==== git submodule status ==== -In the example provided, if 'a' is updated and 'super' is supposed -to see the latest SHA-1 (considering here 'a' has only the master -branch), one should then commit the modified gitlink for 'a' in +In the example provided, if 'sub' is updated and 'super' is supposed +to see the latest SHA-1 (considering here 'sub' has only the master +branch), one should then commit the modified gitlink for 'sub' in the 'super' project. Actually it would not even need to be an -external update, one could move to 'a' folder (insider 'super'), +external update, one could move to 'sub' folder (inside 'super'), modify its content, commit, then move back to 'super' and -commit the modified gitlink for 'a'. +commit the modified gitlink for 'sub'. -== Creating a New Subscription +== Creating a new subscription -=== Defining the Submodule Branch +=== Ensure the subscription is allowed -This is required because submodule subscription is actually the -subscription of a submodule project and one of its branches for -a branch of a super project. +Gerrit has a complex access control system, where different repositories +can be accessed by different groups of people. To ensure that the submodule +related information is allowed to be exposed in the superproject, +the submodule needs to be configured to enable the superproject subscription. +In a submodule client, checkout the refs/meta/config branch and edit +the subscribe capabilities in the 'project.config' file: +==== + git fetch refs/meta/config:refs/meta/config + git checkout refs/meta/config + $EDITOR project.config +==== +and add the following lines: +==== + [subscribe ""] + refs = +==== +where the 'superproject' should be the exact project name of the superproject. +The refspec defines which branches of the submodule are allowed to be +subscribed to which branches of the superproject. See below for +link:#acl_refspec[details]. Push the configuration for review and +submit the change: +==== + git add project.config + git commit -m "Allow to subscribe" + git push HEAD:refs/for/refs/meta/config +==== +After the change is integrated a superproject subscription is possible. + +=== Defining the submodule branch Since Gerrit manages subscriptions in the branch scope, we could have a scenario having a project called 'super' having a branch 'integration' -subscribed to a project called 'a' in branch 'integration', and also -having the same 'super' project but in branch 'dev' subscribed to the 'a' +subscribed to a project called 'sub' in branch 'integration', and also +having the same 'super' project but in branch 'dev' subscribed to the 'sub' project in a branch called 'local-dev'. After adding the git submodule to a super project, one should edit the .gitmodules file to add a branch field to each submodule section which is supposed to be subscribed. -As the branch field is a Gerrit specific field it will not be filled +As the branch field is a Gerrit-specific field it will not be filled automatically by the git submodule command, so one needs to edit it manually. Its value should indicate the branch of a submodule project that when updated will trigger automatic update of its registered @@ -94,28 +130,38 @@ If a git submodule is added but the branch field is not added to the .gitmodules file, Gerrit will not create a subscription for the submodule and there will be no automatic updates to the superproject. -=== Detecting and Subscribing Submodules +Whenever a commit is merged to a project, its project config is checked +to see if any potential superprojects are allowed to subscribe to it. +If so, the superproject is checked if a valid subscription exists +by checking the .gitmodules file for the a submodule which includes +a `branch` field and a url pointing to this server. -Whenever a commit is merged to a project, its content is scanned -to identify if it registers any submodules (if the commit contains new -gitlinks and a .gitmodules file with all required info) and if so, -a new submodule subscription is registered. +[[acl_refspec]] +=== The RefSpec in the allowSuperproject section +The RefSpec for defining the branch level access for subscriptions look similar +to Git style RefSpecs used for pushing in Git. Regular expressions +as found in the ACL configuration are not supported. The most restrictive +RefSpec is allowing one specific branch of the submodule to be subscribed +to one specific branch of the superproject via: +==== + [allowSuperproject ""] + refs = refs/heads/:refs/heads/ +==== -[[automatic_update]] -== Automatic Update of Superprojects +If you want to allow for a 1:1 mapping, i.e. 'master' maps to 'master', +'stable' maps to 'stable', but not allowing 'master' to be subscribed to +'stable': +==== + [allowSuperproject ""] + refs/heads/*:refs/heads/* +==== -After a superproject is subscribed to a submodule, it is not -required to push/merge commits to this superproject to update the -gitlink to the submodule. - -Whenever a commit is merged in a submodule, its subscribed superproject -is updated. - -Imagine a superproject called 'super' having a branch called 'dev' -having subscribed to a submodule 'a' on a branch 'dev-of-a'. When a commit -is merged in branch 'dev-of-a' of 'a' project, Gerrit automatically -creates a new commit on branch 'dev' of 'super' updating the gitlink -to point to the just merged commit. +If you want to enable a branch to be subscribed to any other branch of +the superproject, omit the second part of the RefSpec: +==== + [allowSuperproject ""] + refs/heads/ +==== === Subscription Limitations @@ -123,7 +169,7 @@ Gerrit will only automatically update superprojects where the submodules are hosted on the same Gerrit instance as the superproject. Gerrit determines this by checking the hostname of the submodule specified in the .gitmodules file and comparing it to the -hostname from the canonical web URL. +hostname from the link:config-gerrit.html#gerrit.canonicalWebUrl[`gerrit.canonicalWebUrl`]. It is currently not possible to use the submodule subscription feature with a canonical web URL hostname that differs from the hostname of @@ -174,10 +220,9 @@ from Gerrit. == Removing Subscriptions -If one has added a submodule subscription and drops it, it is -required to merge a commit updating the subscribed super -project/branch to remove the gitlink and the submodule section -of the .gitmodules file. +To remove a subscription, either disable the subscription from the +submodules configuration or remove the submodule or information thereof +(such as the branch field) in the superproject. GERRIT ------ diff --git a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java index 53412cbbd8..24ddbf20bb 100644 --- a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java +++ b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/AbstractSubmoduleSubscription.java @@ -18,7 +18,10 @@ import static com.google.common.truth.Truth.assertThat; import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.common.data.Permission; +import com.google.gerrit.common.data.SubscribeSection; import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.git.MetaDataUpdate; +import com.google.gerrit.server.git.ProjectConfig; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.Config; @@ -69,6 +72,27 @@ public abstract class AbstractSubmoduleSubscription extends AbstractDaemonTest { pushSubmoduleConfig(repo, branch, config); } + protected void allowSubmoduleSubscription(String submodule, String subBranch, + String superproject, String superBranch) throws Exception { + Project.NameKey sub = new Project.NameKey(name(submodule)); + Project.NameKey superName = new Project.NameKey(name(superproject)); + try (MetaDataUpdate md = metaDataUpdateFactory.create(sub)) { + md.setMessage("Added superproject subscription"); + ProjectConfig pc = ProjectConfig.read(md); + SubscribeSection s = new SubscribeSection(superName); + if (superBranch == null) { + s.addRefSpec(subBranch); + } else { + s.addRefSpec(subBranch + ":" + superBranch); + } + pc.addSubscribeSection(s); + ObjectId oldId = pc.getRevision(); + ObjectId newId = pc.commit(md); + assertThat(newId).isNotEqualTo(oldId); + projectCache.evict(pc.getProject()); + } + } + protected void prepareSubmoduleConfigEntry(Config config, String subscribeToRepo, String subscribeToBranch) { subscribeToRepo = name(subscribeToRepo); diff --git a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/SubmoduleSubscriptionsIT.java b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/SubmoduleSubscriptionsIT.java index ae378212b1..e69a64708e 100644 --- a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/SubmoduleSubscriptionsIT.java +++ b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/SubmoduleSubscriptionsIT.java @@ -43,8 +43,11 @@ public class SubmoduleSubscriptionsIT extends AbstractSubmoduleSubscription { public void testSubscriptionToEmptyRepo() throws Exception { TestRepository superRepo = createProjectWithPush("super-project"); TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", "refs/heads/master"); createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); + pushChangeTo(subRepo, "master"); ObjectId subHEAD = pushChangeTo(subRepo, "master"); expectToHaveSubmoduleState(superRepo, "master", "subscribed-to-project", subHEAD); @@ -54,6 +57,8 @@ public class SubmoduleSubscriptionsIT extends AbstractSubmoduleSubscription { public void testSubscriptionToExistingRepo() throws Exception { TestRepository superRepo = createProjectWithPush("super-project"); TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", "refs/heads/master"); pushChangeTo(subRepo, "master"); createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); @@ -62,11 +67,69 @@ public class SubmoduleSubscriptionsIT extends AbstractSubmoduleSubscription { "subscribed-to-project", subHEAD); } + @Test + public void testSubscriptionWildcardACLForSingleBranch() throws Exception { + TestRepository superRepo = createProjectWithPush("super-project"); + TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + // master is allowed to be subscribed to any superprojects branch: + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", null); + // create 'branch': + pushChangeTo(superRepo, "branch"); + + pushChangeTo(subRepo, "master"); + createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); + createSubmoduleSubscription(superRepo, "branch", "subscribed-to-project", "master"); + + ObjectId subHEAD = pushChangeTo(subRepo, "master"); + + expectToHaveSubmoduleState(superRepo, "master", + "subscribed-to-project", subHEAD); + expectToHaveSubmoduleState(superRepo, "branch", + "subscribed-to-project", subHEAD); + } + + @Test + public void testSubscriptionWildcardACLOneOnOneMapping() throws Exception { + TestRepository superRepo = createProjectWithPush("super-project"); + TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + // any branch is allowed to be subscribed to the same superprojects branch: + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/*", + "super-project", "refs/heads/*"); + + // create 'branch' in both repos: + pushChangeTo(superRepo, "branch"); + pushChangeTo(subRepo, "branch"); + + pushChangeTo(subRepo, "master"); + createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); + createSubmoduleSubscription(superRepo, "branch", "subscribed-to-project", "branch"); + + ObjectId subHEAD1 = pushChangeTo(subRepo, "master"); + ObjectId subHEAD2 = pushChangeTo(subRepo, "branch"); + + expectToHaveSubmoduleState(superRepo, "master", + "subscribed-to-project", subHEAD1); + expectToHaveSubmoduleState(superRepo, "branch", + "subscribed-to-project", subHEAD2); + + // Now test that cross subscriptions do not work: + createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "branch"); + ObjectId subHEAD3 = pushChangeTo(subRepo, "branch"); + + expectToHaveSubmoduleState(superRepo, "master", + "subscribed-to-project", subHEAD1); + expectToHaveSubmoduleState(superRepo, "branch", + "subscribed-to-project", subHEAD3); + } + @Test @GerritConfig(name = "submodule.verboseSuperprojectUpdate", value = "false") public void testSubmoduleShortCommitMessage() throws Exception { TestRepository superRepo = createProjectWithPush("super-project"); TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", "refs/heads/master"); pushChangeTo(subRepo, "master"); createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); @@ -90,6 +153,8 @@ public class SubmoduleSubscriptionsIT extends AbstractSubmoduleSubscription { public void testSubmoduleCommitMessage() throws Exception { TestRepository superRepo = createProjectWithPush("super-project"); TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", "refs/heads/master"); pushChangeTo(subRepo, "master"); createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); @@ -118,6 +183,8 @@ public class SubmoduleSubscriptionsIT extends AbstractSubmoduleSubscription { public void testSubscriptionUnsubscribe() throws Exception { TestRepository superRepo = createProjectWithPush("super-project"); TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", "refs/heads/master"); pushChangeTo(subRepo, "master"); createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); @@ -141,6 +208,8 @@ public class SubmoduleSubscriptionsIT extends AbstractSubmoduleSubscription { public void testSubscriptionUnsubscribeByDeletingGitModules() throws Exception { TestRepository superRepo = createProjectWithPush("super-project"); TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", "refs/heads/master"); pushChangeTo(subRepo, "master"); createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); @@ -164,6 +233,8 @@ public class SubmoduleSubscriptionsIT extends AbstractSubmoduleSubscription { public void testSubscriptionToDifferentBranches() throws Exception { TestRepository superRepo = createProjectWithPush("super-project"); TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/foo", + "super-project", "refs/heads/master"); createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "foo"); ObjectId subFoo = pushChangeTo(subRepo, "foo"); @@ -177,6 +248,10 @@ public class SubmoduleSubscriptionsIT extends AbstractSubmoduleSubscription { public void testCircularSubscriptionIsDetected() throws Exception { TestRepository superRepo = createProjectWithPush("super-project"); TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", "refs/heads/master"); + allowSubmoduleSubscription("super-project", "refs/heads/master", + "subscribed-to-project", "refs/heads/master"); pushChangeTo(subRepo, "master"); createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); @@ -191,6 +266,44 @@ public class SubmoduleSubscriptionsIT extends AbstractSubmoduleSubscription { assertThat(hasSubmodule(subRepo, "master", "super-project")).isFalse(); } + + @Test + public void testSubscriptionFailOnMissingACL() throws Exception { + TestRepository superRepo = createProjectWithPush("super-project"); + TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + + pushChangeTo(subRepo, "master"); + createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); + pushChangeTo(subRepo, "master"); + assertThat(hasSubmodule(superRepo, "master", "subscribed-to-project")).isFalse(); + } + + @Test + public void testSubscriptionFailOnWrongProjectACL() throws Exception { + TestRepository superRepo = createProjectWithPush("super-project"); + TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "wrong-super-project", "refs/heads/master"); + + pushChangeTo(subRepo, "master"); + createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); + pushChangeTo(subRepo, "master"); + assertThat(hasSubmodule(superRepo, "master", "subscribed-to-project")).isFalse(); + } + + @Test + public void testSubscriptionFailOnWrongBranchACL() throws Exception { + TestRepository superRepo = createProjectWithPush("super-project"); + TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", "refs/heads/wrong-branch"); + + pushChangeTo(subRepo, "master"); + createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); + pushChangeTo(subRepo, "master"); + assertThat(hasSubmodule(superRepo, "master", "subscribed-to-project")).isFalse(); + } + private void deleteAllSubscriptions(TestRepository repo, String branch) throws Exception { repo.git().fetch().setRemote("origin").call(); diff --git a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/SubmoduleSubscriptionsWholeTopicMergeIT.java b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/SubmoduleSubscriptionsWholeTopicMergeIT.java index e4a054a3fa..30482dd56b 100644 --- a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/SubmoduleSubscriptionsWholeTopicMergeIT.java +++ b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/SubmoduleSubscriptionsWholeTopicMergeIT.java @@ -41,6 +41,9 @@ public class SubmoduleSubscriptionsWholeTopicMergeIT public void testSubscriptionUpdateOfManyChanges() throws Exception { TestRepository superRepo = createProjectWithPush("super-project"); TestRepository subRepo = createProjectWithPush("subscribed-to-project"); + allowSubmoduleSubscription("subscribed-to-project", "refs/heads/master", + "super-project", "refs/heads/master"); + createSubmoduleSubscription(superRepo, "master", "subscribed-to-project", "master"); ObjectId subHEAD = subRepo.branch("HEAD").commit().insertChangeId() @@ -96,6 +99,13 @@ public class SubmoduleSubscriptionsWholeTopicMergeIT TestRepository sub2 = createProjectWithPush("sub2"); TestRepository sub3 = createProjectWithPush("sub3"); + allowSubmoduleSubscription("sub1", "refs/heads/master", + "super-project", "refs/heads/master"); + allowSubmoduleSubscription("sub2", "refs/heads/master", + "super-project", "refs/heads/master"); + allowSubmoduleSubscription("sub3", "refs/heads/master", + "super-project", "refs/heads/master"); + Config config = new Config(); prepareSubmoduleConfigEntry(config, "sub1", "master"); prepareSubmoduleConfigEntry(config, "sub2", "master"); diff --git a/gerrit-common/BUCK b/gerrit-common/BUCK index ab62b40b72..ac36cc7d89 100644 --- a/gerrit-common/BUCK +++ b/gerrit-common/BUCK @@ -12,6 +12,7 @@ EXCLUDES = [ SRC + 'common/FileUtil.java', SRC + 'common/IoUtil.java', SRC + 'common/TimeUtil.java', + SRC + 'common/data/SubscribeSection.java', ] java_library( diff --git a/gerrit-common/src/main/java/com/google/gerrit/common/data/SubscribeSection.java b/gerrit-common/src/main/java/com/google/gerrit/common/data/SubscribeSection.java new file mode 100644 index 0000000000..7ec1edaba8 --- /dev/null +++ b/gerrit-common/src/main/java/com/google/gerrit/common/data/SubscribeSection.java @@ -0,0 +1,69 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.common.data; + +import com.google.gerrit.reviewdb.client.Branch; +import com.google.gerrit.reviewdb.client.Project; + +import org.eclipse.jgit.transport.RefSpec; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** Portion of a {@link Project} describing superproject subscription rules. */ +public class SubscribeSection { + + private final List refSpecs; + private final Project.NameKey project; + + public SubscribeSection(Project.NameKey p) { + project = p; + refSpecs = new ArrayList<>(); + } + + public void addRefSpec(RefSpec spec) { + refSpecs.add(spec); + } + + public void addRefSpec(String spec) { + refSpecs.add(new RefSpec(spec)); + } + + public Project.NameKey getProject() { + return project; + } + + /** + * Determines if the branch could trigger a + * superproject update as allowed via this subscribe section. + * + * @param branch the branch to check + * @return if the branch could trigger a superproject update + */ + public boolean appliesTo(Branch.NameKey branch) { + for (RefSpec r : refSpecs) { + if (r.matchSource(branch.get())) { + return true; + } + } + return false; + } + + public Collection getRefSpecs() { + return Collections.unmodifiableCollection(refSpecs); + } +} diff --git a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/ReviewDb.java b/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/ReviewDb.java index 6232f2a376..5d073e24c9 100644 --- a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/ReviewDb.java +++ b/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/ReviewDb.java @@ -96,8 +96,7 @@ public interface ReviewDb extends Schema { @Relation(id = 26) PatchLineCommentAccess patchComments(); - @Relation(id = 28) - SubmoduleSubscriptionAccess submoduleSubscriptions(); + // Deleted @Relation(id = 28) @Relation(id = 29) AccountGroupByIdAccess accountGroupById(); diff --git a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/ReviewDbWrapper.java b/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/ReviewDbWrapper.java index 7522c2df2d..deecba96a1 100644 --- a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/ReviewDbWrapper.java +++ b/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/ReviewDbWrapper.java @@ -153,11 +153,6 @@ public class ReviewDbWrapper implements ReviewDb { return delegate.patchComments(); } - @Override - public SubmoduleSubscriptionAccess submoduleSubscriptions() { - return delegate.submoduleSubscriptions(); - } - @Override public AccountGroupByIdAccess accountGroupById() { return delegate.accountGroupById(); diff --git a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/SubmoduleSubscriptionAccess.java b/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/SubmoduleSubscriptionAccess.java deleted file mode 100644 index b25e4066c0..0000000000 --- a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/server/SubmoduleSubscriptionAccess.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (C) 2011 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.gerrit.reviewdb.server; - -import com.google.gerrit.reviewdb.client.Branch; -import com.google.gerrit.reviewdb.client.Project; -import com.google.gerrit.reviewdb.client.SubmoduleSubscription; -import com.google.gwtorm.server.Access; -import com.google.gwtorm.server.OrmException; -import com.google.gwtorm.server.PrimaryKey; -import com.google.gwtorm.server.Query; -import com.google.gwtorm.server.ResultSet; - -public interface SubmoduleSubscriptionAccess extends - Access { - @Override - @PrimaryKey("key") - SubmoduleSubscription get(SubmoduleSubscription.Key key) throws OrmException; - - @Query("WHERE key.superProject = ?") - ResultSet bySuperProject(Branch.NameKey superProject) - throws OrmException; - - /** - * Fetches all {@code SubmoduleSubscription}s in which some branch of - * {@code superProject} subscribes a branch. - * - * Use {@link #bySuperProject(Branch.NameKey)} to fetch for a branch instead - * of a project. - * - * @param superProject the project to fetch subscriptions for - * @return {@code SubmoduleSubscription}s that are subscribed by some - * branch of {@code superProject}. - * @throws OrmException - */ - @Query("WHERE key.superProject.projectName = ?") - ResultSet bySuperProjectProject(Project.NameKey superProject) - throws OrmException; - - @Query("WHERE submodule = ?") - ResultSet bySubmodule(Branch.NameKey submodule) - throws OrmException; - - /** - * Fetches all {@code SubmoduleSubscription}s in which some branch of - * {@code submodule} is subscribed. - * - * Use {@link #bySubmodule(Branch.NameKey)} to fetch for a branch instead of - * a project. - * - * @param submodule the project to fetch subscriptions for. - * @return {@code SubmoduleSubscription}s that subscribe some branch of - * {@code submodule}. - * @throws OrmException - */ - @Query("WHERE submodule.projectName = ?") - ResultSet bySubmoduleProject(Project.NameKey submodule) - throws OrmException; -} diff --git a/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_generic.sql b/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_generic.sql index 1162a5f15e..b8ebdde9bf 100644 --- a/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_generic.sql +++ b/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_generic.sql @@ -93,9 +93,3 @@ ON patch_sets (revision); CREATE INDEX starred_changes_byChange ON starred_changes (change_id); - --- ********************************************************************* --- SubmoduleSubscriptionAccess - -CREATE INDEX submodule_subscr_acc_byS -ON submodule_subscriptions (submodule_project_name, submodule_branch_name); diff --git a/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_maxdb.sql b/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_maxdb.sql index 258b7be2d6..d7135a217b 100644 --- a/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_maxdb.sql +++ b/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_maxdb.sql @@ -103,11 +103,3 @@ ON patch_sets (revision) CREATE INDEX starred_changes_byChange ON starred_changes (change_id) # - --- ********************************************************************* --- SubmoduleSubscriptionAccess - -CREATE INDEX submod_subscr_ac_bySubscription -ON submodule_subscriptions (submodule_project_name, submodule_branch_name) -# - diff --git a/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_postgres.sql b/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_postgres.sql index 1fe7dce031..b0b6b6f143 100644 --- a/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_postgres.sql +++ b/gerrit-reviewdb/src/main/resources/com/google/gerrit/reviewdb/server/index_postgres.sql @@ -143,8 +143,3 @@ ON patch_sets (revision); CREATE INDEX starred_changes_byChange ON starred_changes (change_id); --- ********************************************************************* --- SubmoduleSubscriptionAccess - -CREATE INDEX submodule_subscr_acc_byS -ON submodule_subscriptions (submodule_project_name, submodule_branch_name); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java index 81b3e0f11a..25c8941741 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java @@ -84,6 +84,7 @@ import com.google.gerrit.server.extensions.events.GitReferenceUpdated; import com.google.gerrit.server.git.BatchUpdate; import com.google.gerrit.server.git.EmailMerge; import com.google.gerrit.server.git.GitModule; +import com.google.gerrit.server.git.GitModules; import com.google.gerrit.server.git.MergeUtil; import com.google.gerrit.server.git.NotesBranchUtil; import com.google.gerrit.server.git.ReceivePackInitializer; @@ -324,6 +325,7 @@ public class GerritGlobalModule extends FactoryModule { factory(NotesBranchUtil.Factory.class); factory(SubmoduleSectionParser.Factory.class); factory(ReplaceOp.Factory.class); + factory(GitModules.Factory.class); bind(AccountManager.class); factory(ChangeUserName.Factory.class); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/GitModules.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/GitModules.java new file mode 100644 index 0000000000..c3ba0d4950 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/GitModules.java @@ -0,0 +1,131 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.git; + +import com.google.gerrit.common.Nullable; +import com.google.gerrit.reviewdb.client.Branch; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.reviewdb.client.SubmoduleSubscription; +import com.google.gerrit.server.config.CanonicalWebUrl; +import com.google.gerrit.server.util.SubmoduleSectionParser; +import com.google.inject.assistedinject.Assisted; +import com.google.inject.assistedinject.AssistedInject; + +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.BlobBasedConfig; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; + +/** + * Loads the .gitmodules file of the specified project/branch. + * It can be queried which submodules this branch is subscribed to. + */ +public class GitModules { + private static final Logger log = LoggerFactory.getLogger(GitModules.class); + + public interface Factory { + GitModules create(Branch.NameKey project, String submissionId); + } + + private static final String GIT_MODULES = ".gitmodules"; + + private final String thisServer; + private final GitRepositoryManager repoManager; + private final SubmoduleSectionParser.Factory subSecParserFactory; + private final Branch.NameKey branch; + private final String submissionId; + + Set subscriptions; + + @AssistedInject + GitModules( + @CanonicalWebUrl @Nullable String canonicalWebUrl, + GitRepositoryManager repoManager, + SubmoduleSectionParser.Factory subSecParserFactory, + @Assisted Branch.NameKey branch, + @Assisted String submissionId) throws SubmoduleException { + this.repoManager = repoManager; + this.subSecParserFactory = subSecParserFactory; + this.branch = branch; + this.submissionId = submissionId; + try { + this.thisServer = new URI(canonicalWebUrl).getHost(); + } catch (URISyntaxException e) { + throw new SubmoduleException("Incorrect Gerrit canonical web url " + + "provided in gerrit.config file.", e); + } + } + + void load() throws SubmoduleException { + Project.NameKey project = branch.getParentKey(); + logDebug("Loading .gitmodules of {} for project {}", branch, project); + try (Repository repo = repoManager.openRepository(project); + RevWalk rw = new RevWalk(repo)) { + + ObjectId id = repo.resolve(branch.get()); + if (id == null) { + throw new IOException("Cannot open branch " + branch.get()); + } + RevCommit commit = rw.parseCommit(id); + + TreeWalk tw = TreeWalk.forPath(repo, GIT_MODULES, commit.getTree()); + if (tw == null + || (tw.getRawMode(0) & FileMode.TYPE_MASK) != FileMode.TYPE_FILE) { + return; + } + + BlobBasedConfig bbc = + new BlobBasedConfig(null, repo, commit, GIT_MODULES); + + subscriptions = subSecParserFactory.create(bbc, thisServer, + branch).parseAllSections(); + } catch (ConfigInvalidException | IOException e) { + throw new SubmoduleException( + "Could not read .gitmodule file of super project: " + + branch.getParentKey(), e); + } + } + + public Collection subscribedTo(Branch.NameKey src) { + logDebug("Checking for a subscription of " + src); + Collection ret = new ArrayList<>(); + for (SubmoduleSubscription s : subscriptions) { + if (s.getSubmodule().equals(src)) { + logDebug("Found " + s); + ret.add(s); + } + } + return ret; + } + + private void logDebug(String msg, Object... args) { + if (log.isDebugEnabled()) { + log.debug("[" + submissionId + "]" + msg, args); + } + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/MergeOp.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/MergeOp.java index e213927279..a58e358da5 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/git/MergeOp.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/MergeOp.java @@ -650,10 +650,6 @@ public class MergeOp implements AutoCloseable { } SubmoduleOp subOp = subOpProvider.get(); - for (Branch.NameKey branch : branches) { - OpenBranch ob = getRepo(branch.getParentKey()).getBranch(branch); - updateSubmoduleSubscriptions(ob, subOp); - } updateSuperProjects(subOp, br.values()); } @@ -854,26 +850,12 @@ public class MergeOp implements AutoCloseable { } } - private void updateSubmoduleSubscriptions(OpenBranch ob, SubmoduleOp subOp) { - CodeReviewCommit branchTip = ob.oldTip; - MergeTip mergeTip = ob.mergeTip; - if (mergeTip != null - && (branchTip == null || branchTip != mergeTip.getCurrentTip())) { - logDebug("Updating submodule subscriptions for branch {}", ob.name); - try { - subOp.updateSubmoduleSubscriptions(db, ob.name); - } catch (SubmoduleException e) { - logError("The submodule subscriptions were not updated according" - + "to the .gitmodules files", e); - } - } - } - private void updateSuperProjects(SubmoduleOp subOp, Collection branches) { logDebug("Updating superprojects"); try { - subOp.updateSuperProjects(db, branches); + subOp.updateSuperProjects(db, branches, submissionId); + logDebug("Updating superprojects done"); } catch (SubmoduleException e) { logError("The gitlinks were not updated according to the " + "subscriptions", e); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/ProjectConfig.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/ProjectConfig.java index 8296afa6bc..41c2858823 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/git/ProjectConfig.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/ProjectConfig.java @@ -39,10 +39,12 @@ import com.google.gerrit.common.data.Permission; import com.google.gerrit.common.data.PermissionRule; import com.google.gerrit.common.data.PermissionRule.Action; import com.google.gerrit.common.data.RefConfigSection; +import com.google.gerrit.common.data.SubscribeSection; import com.google.gerrit.extensions.client.InheritableBoolean; import com.google.gerrit.extensions.client.ProjectState; import com.google.gerrit.extensions.client.SubmitType; import com.google.gerrit.reviewdb.client.AccountGroup; +import com.google.gerrit.reviewdb.client.Branch; import com.google.gerrit.reviewdb.client.AccountProjectWatch.NotifyType; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.client.RefNames; @@ -56,6 +58,7 @@ import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.util.StringUtils; import java.io.IOException; @@ -125,6 +128,9 @@ public class ProjectConfig extends VersionedMetaData implements ValidationError. private static final String KEY_MERGE_CONTENT = "mergeContent"; private static final String KEY_STATE = "state"; + private static final String SUBSCRIBE_SECTION = "allowSuperproject"; + private static final String SUBSCRIBE_REFS = "refs"; + private static final String DASHBOARD = "dashboard"; private static final String KEY_DEFAULT = "default"; private static final String KEY_LOCAL_DEFAULT = "local-default"; @@ -161,6 +167,7 @@ public class ProjectConfig extends VersionedMetaData implements ValidationError. private Map notifySections; private Map labelSections; private ConfiguredMimeTypes mimeTypes; + private Map subscribeSections; private List commentLinkSections; private List validationErrors; private ObjectId rulesId; @@ -255,6 +262,21 @@ public class ProjectConfig extends VersionedMetaData implements ValidationError. return branchOrderSection; } + public Collection getSubscribeSections( + Branch.NameKey branch) { + Collection ret = new ArrayList<>(); + for (SubscribeSection s : subscribeSections.values()) { + if (s.appliesTo(branch)) { + ret.add(s); + } + } + return ret; + } + + public void addSubscribeSection(SubscribeSection s) { + subscribeSections.put(s.getProject(), s); + } + public void remove(AccessSection section) { if (section != null) { accessSections.remove(section.getName()); @@ -440,6 +462,7 @@ public class ProjectConfig extends VersionedMetaData implements ValidationError. loadNotifySections(rc, groupsByName); loadLabelSections(rc); loadCommentLinkSections(rc); + loadSubscribeSections(rc); mimeTypes = new ConfiguredMimeTypes(projectName.get(), rc); loadPluginSections(rc); loadReceiveSection(rc); @@ -771,6 +794,24 @@ public class ProjectConfig extends VersionedMetaData implements ValidationError. commentLinkSections = ImmutableList.copyOf(commentLinkSections); } + private void loadSubscribeSections(Config rc) throws ConfigInvalidException { + Set subsections = rc.getSubsections(SUBSCRIBE_SECTION); + subscribeSections = new HashMap<>(); + try { + for (String projectName : subsections) { + Project.NameKey p = new Project.NameKey(projectName); + SubscribeSection ss = new SubscribeSection(p); + for (String s : rc.getStringList(SUBSCRIBE_SECTION, + projectName, SUBSCRIBE_REFS)) { + ss.addRefSpec(s); + } + subscribeSections.put(p, ss); + } + } catch (IllegalArgumentException e) { + throw new ConfigInvalidException(e.getMessage()); + } + } + private void loadReceiveSection(Config rc) { checkReceivedObjects = rc.getBoolean(RECEIVE, KEY_CHECK_RECEIVED_OBJECTS, true); maxObjectSizeLimit = rc.getLong(RECEIVE, null, KEY_MAX_OBJECT_SIZE_LIMIT, 0); @@ -865,6 +906,7 @@ public class ProjectConfig extends VersionedMetaData implements ValidationError. savePluginSections(rc, keepGroups); groupList.retainUUIDs(keepGroups); saveLabelSections(rc); + saveSubscribeSections(rc); saveConfig(PROJECT_CONFIG, rc); saveGroupList(); @@ -1147,6 +1189,15 @@ public class ProjectConfig extends VersionedMetaData implements ValidationError. saveUTF8(GroupList.FILE_NAME, groupList.asText()); } + private void saveSubscribeSections(Config rc) { + for (Project.NameKey p : subscribeSections.keySet()) { + SubscribeSection s = subscribeSections.get(p); + for (RefSpec r : s.getRefSpecs()) { + rc.setString(SUBSCRIBE_SECTION, p.get(), SUBSCRIBE_REFS, r.toString()); + } + } + } + private > E getEnum(Config rc, String section, String subsection, String name, E defaultValue) { try { diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java index 775ecae381..fd4400d824 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java @@ -79,7 +79,6 @@ import com.google.gerrit.reviewdb.client.PatchSetApproval; import com.google.gerrit.reviewdb.client.PatchSetInfo; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.client.RefNames; -import com.google.gerrit.reviewdb.client.SubmoduleSubscription; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.ApprovalsUtil; import com.google.gerrit.server.ChangeMessagesUtil; @@ -127,7 +126,6 @@ import com.google.gerrit.server.util.MagicBranch; import com.google.gerrit.server.util.RequestScopePropagator; import com.google.gerrit.util.cli.CmdLineParser; import com.google.gwtorm.server.OrmException; -import com.google.gwtorm.server.ResultSet; import com.google.gwtorm.server.SchemaFactory; import com.google.inject.Inject; import com.google.inject.Provider; @@ -643,17 +641,6 @@ public class ReceiveCommits { break; case DELETE: - ResultSet submoduleSubscriptions = null; - Branch.NameKey projRef = new Branch.NameKey(project.getNameKey(), - c.getRefName()); - try { - submoduleSubscriptions = - db.submoduleSubscriptions().bySuperProject(projRef); - db.submoduleSubscriptions().delete(submoduleSubscriptions); - } catch (OrmException e) { - log.error("Cannot delete submodule subscription(s) of branch " - + projRef + ": " + submoduleSubscriptions, e); - } break; } } @@ -681,11 +668,9 @@ public class ReceiveCommits { // Update superproject gitlinks if required. SubmoduleOp op = subOpProvider.get(); try { - op.updateSubmoduleSubscriptions(db, branches); - op.updateSuperProjects(db, branches); + op.updateSuperProjects(db, branches, "receiveID"); } catch (SubmoduleException e) { - log.error("Can't update submodule subscriptions " - + "or update the superprojects", e); + log.error("Can't update the superprojects", e); } closeProgress.end(); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/SubmoduleOp.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/SubmoduleOp.java index f7b447969c..51c41710cc 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/git/SubmoduleOp.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/SubmoduleOp.java @@ -19,18 +19,18 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import com.google.gerrit.common.ChangeHooks; import com.google.gerrit.common.Nullable; +import com.google.gerrit.common.data.SubscribeSection; import com.google.gerrit.reviewdb.client.Account; import com.google.gerrit.reviewdb.client.Branch; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.reviewdb.client.RefNames; import com.google.gerrit.reviewdb.client.SubmoduleSubscription; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.GerritPersonIdent; -import com.google.gerrit.server.config.CanonicalWebUrl; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.extensions.events.GitReferenceUpdated; -import com.google.gerrit.server.util.SubmoduleSectionParser; -import com.google.gwtorm.server.OrmException; +import com.google.gerrit.server.project.ProjectCache; import com.google.inject.Inject; -import com.google.inject.Provider; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheBuilder; @@ -38,10 +38,8 @@ import org.eclipse.jgit.dircache.DirCacheEditor; import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; import org.eclipse.jgit.dircache.DirCacheEntry; -import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.lib.BlobBasedConfig; import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Constants; @@ -54,176 +52,141 @@ import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.transport.RefSpec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.Set; public class SubmoduleOp { private static final Logger log = LoggerFactory.getLogger(SubmoduleOp.class); - private static final String GIT_MODULES = ".gitmodules"; - private final Provider urlProvider; + private final GitModules.Factory gitmodulesFactory; private final PersonIdent myIdent; private final GitRepositoryManager repoManager; private final GitReferenceUpdated gitRefUpdated; + private final ProjectCache projectCache; private final Set updatedSubscribers; private final Account account; private final ChangeHooks changeHooks; - private final SubmoduleSectionParser.Factory subSecParserFactory; private final boolean verboseSuperProject; private final boolean enableSuperProjectSubscriptions; + private String updateId; @Inject public SubmoduleOp( - @CanonicalWebUrl @Nullable Provider urlProvider, + GitModules.Factory gitmodulesFactory, @GerritPersonIdent PersonIdent myIdent, @GerritServerConfig Config cfg, GitRepositoryManager repoManager, GitReferenceUpdated gitRefUpdated, + ProjectCache projectCache, @Nullable Account account, - ChangeHooks changeHooks, - SubmoduleSectionParser.Factory subSecParserFactory) { - this.urlProvider = urlProvider; + ChangeHooks changeHooks) { + this.gitmodulesFactory = gitmodulesFactory; this.myIdent = myIdent; this.repoManager = repoManager; this.gitRefUpdated = gitRefUpdated; + this.projectCache = projectCache; this.account = account; this.changeHooks = changeHooks; - this.subSecParserFactory = subSecParserFactory; this.verboseSuperProject = cfg.getBoolean("submodule", "verboseSuperprojectUpdate", true); this.enableSuperProjectSubscriptions = cfg.getBoolean("submodule", "enableSuperProjectSubscriptions", true); updatedSubscribers = new HashSet<>(); + } - void updateSubmoduleSubscriptions(ReviewDb db, Set branches) - throws SubmoduleException { - if (!enableSuperProjectSubscriptions) { - return; - } - - for (Branch.NameKey branch : branches) { - updateSubmoduleSubscriptions(db, branch); - } - } - - void updateSubmoduleSubscriptions(ReviewDb db, Branch.NameKey destBranch) - throws SubmoduleException { - if (!enableSuperProjectSubscriptions) { - return; - } - if (urlProvider.get() == null) { - logAndThrowSubmoduleException("Cannot establish canonical web url used " - + "to access gerrit. It should be provided in gerrit.config file."); - } - try (Repository repo = repoManager.openRepository( - destBranch.getParentKey()); - RevWalk rw = new RevWalk(repo)) { - - ObjectId id = repo.resolve(destBranch.get()); - if (id == null) { - logAndThrowSubmoduleException( - "Cannot resolve submodule destination branch " + destBranch); - } - RevCommit commit = rw.parseCommit(id); - - Set oldSubscriptions = - Sets.newHashSet(db.submoduleSubscriptions() - .bySuperProject(destBranch)); - - Set newSubscriptions; - TreeWalk tw = TreeWalk.forPath(repo, GIT_MODULES, commit.getTree()); - if (tw != null - && (FileMode.REGULAR_FILE.equals(tw.getRawMode(0)) || - FileMode.EXECUTABLE_FILE.equals(tw.getRawMode(0)))) { - BlobBasedConfig bbc = - new BlobBasedConfig(null, repo, commit, GIT_MODULES); - - String thisServer = new URI(urlProvider.get()).getHost(); - - newSubscriptions = subSecParserFactory.create(bbc, thisServer, - destBranch).parseAllSections(); - } else { - newSubscriptions = Collections.emptySet(); - } - - Set alreadySubscribeds = new HashSet<>(); - for (SubmoduleSubscription s : newSubscriptions) { - if (oldSubscriptions.contains(s)) { - alreadySubscribeds.add(s); + public Collection getDestinationBranches(Branch.NameKey src, + SubscribeSection s) throws IOException { + Collection ret = new ArrayList<>(); + logDebug("Inspecting SubscribeSection " + s); + for (RefSpec r : s.getRefSpecs()) { + logDebug("Inspecting ref " + r); + if (r.matchSource(src.get())) { + if (r.getDestination() == null) { + // no need to care for wildcard, as we matched already + try (Repository repo = repoManager.openRepository(s.getProject())) { + for (Ref ref : repo.getRefDatabase().getRefs( + RefNames.REFS_HEADS).values()) { + ret.add(new Branch.NameKey(s.getProject(), ref.getName())); + } + } + } else if (r.isWildcard()) { + // refs/heads/*:refs/heads/* + ret.add(new Branch.NameKey(s.getProject(), + r.expandFromSource(src.get()).getDestination())); + } else { + // e.g. refs/heads/master:refs/heads/stable + ret.add(new Branch.NameKey(s.getProject(), r.getDestination())); } } - - oldSubscriptions.removeAll(newSubscriptions); - newSubscriptions.removeAll(alreadySubscribeds); - - if (!oldSubscriptions.isEmpty()) { - db.submoduleSubscriptions().delete(oldSubscriptions); - } - if (!newSubscriptions.isEmpty()) { - db.submoduleSubscriptions().insert(newSubscriptions); - } - - } catch (OrmException e) { - logAndThrowSubmoduleException( - "Database problem at update of subscriptions table from " - + GIT_MODULES + " file.", e); - } catch (ConfigInvalidException e) { - logAndThrowSubmoduleException( - "Problem at update of subscriptions table: " + GIT_MODULES - + " config file is invalid.", e); - } catch (IOException e) { - logAndThrowSubmoduleException( - "Problem at update of subscriptions table from " + GIT_MODULES + ".", - e); - } catch (URISyntaxException e) { - logAndThrowSubmoduleException( - "Incorrect gerrit canonical web url provided in gerrit.config file.", - e); } + logDebug("Returning possible branches: " + ret + + "for project " + s.getProject()); + return ret; + } + + private Collection + superProjectSubscriptionsForSubmoduleBranch( + Branch.NameKey branch) throws SubmoduleException { + logDebug("Calculating possible superprojects for " + branch); + Collection ret = new ArrayList<>(); + Project.NameKey project = branch.getParentKey(); + ProjectConfig cfg = projectCache.get(project).getConfig(); + try { + for (SubscribeSection s : cfg.getSubscribeSections(branch)) { + Collection branches = getDestinationBranches(branch, s); + for (Branch.NameKey targetBranch : branches) { + GitModules m = gitmodulesFactory.create(targetBranch, updateId); + m.load(); + ret.addAll(m.subscribedTo(branch)); + } + } + } catch (IOException e) { + throw new SubmoduleException("Could not update superproject", e); + } + logDebug("Calculated superprojects for " + branch + " are "+ ret); + return ret; } protected void updateSuperProjects(ReviewDb db, - Collection updatedBranches) throws SubmoduleException { + Collection updatedBranches, String updateId) + throws SubmoduleException { if (!enableSuperProjectSubscriptions) { + logDebug("Updating superprojects disabled"); return; } - try { - // These (repo/branch) will be updated later with all the given - // individual submodule subscriptions - Multimap targets = - HashMultimap.create(); + this.updateId = updateId; + logDebug("Updating superprojects"); + // These (repo/branch) will be updated later with all the given + // individual submodule subscriptions + Multimap targets = + HashMultimap.create(); - for (Branch.NameKey updatedBranch : updatedBranches) { - for (SubmoduleSubscription sub : db.submoduleSubscriptions() - .bySubmodule(updatedBranch)) { - targets.put(sub.getSuperProject(), sub); - } + for (Branch.NameKey updatedBranch : updatedBranches) { + for (SubmoduleSubscription sub : + superProjectSubscriptionsForSubmoduleBranch(updatedBranch)) { + targets.put(sub.getSuperProject(), sub); } - updatedSubscribers.addAll(updatedBranches); - // Update subscribers. - for (Branch.NameKey dest : targets.keySet()) { - try { - if (!updatedSubscribers.add(dest)) { - log.error("Possible circular subscription involving " + dest); - } else { - updateGitlinks(db, dest, targets.get(dest)); - } - } catch (SubmoduleException e) { - log.warn("Cannot update gitlinks for " + dest, e); + } + updatedSubscribers.addAll(updatedBranches); + // Update subscribers. + for (Branch.NameKey dest : targets.keySet()) { + try { + if (!updatedSubscribers.add(dest)) { + log.error("Possible circular subscription involving " + dest); + } else { + updateGitlinks(db, dest, targets.get(dest)); } + } catch (SubmoduleException e) { + log.warn("Cannot update gitlinks for " + dest, e); } - } catch (OrmException e) { - logAndThrowSubmoduleException("Cannot read subscription records", e); } } @@ -304,7 +267,7 @@ public class SubmoduleOp { msgbuf.append(c.getFullMessage() + "\n\n"); } } catch (IOException e) { - logAndThrowSubmoduleException("Could not perform a revwalk to " + throw new SubmoduleException("Could not perform a revwalk to " + "create superproject commit message", e); } } @@ -359,7 +322,7 @@ public class SubmoduleOp { throw new IOException(rfu.getResult().name()); } // Recursive call: update subscribers of the subscriber - updateSuperProjects(db, Sets.newHashSet(subscriber)); + updateSuperProjects(db, Sets.newHashSet(subscriber), updateId); } catch (IOException e) { throw new SubmoduleException("Cannot update gitlinks for " + subscriber.get(), e); @@ -379,15 +342,9 @@ public class SubmoduleOp { } } - private static void logAndThrowSubmoduleException(final String errorMsg, - final Exception e) throws SubmoduleException { - log.error(errorMsg, e); - throw new SubmoduleException(errorMsg, e); - } - - private static void logAndThrowSubmoduleException(final String errorMsg) - throws SubmoduleException { - log.error(errorMsg); - throw new SubmoduleException(errorMsg); + private void logDebug(String msg, Object... args) { + if (log.isDebugEnabled()) { + log.debug("[" + updateId + "]" + msg, args); + } } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/DeleteBranch.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/DeleteBranch.java index 7da38ed64d..bda2c710f1 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/project/DeleteBranch.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/DeleteBranch.java @@ -19,15 +19,12 @@ import com.google.gerrit.extensions.restapi.AuthException; import com.google.gerrit.extensions.restapi.ResourceConflictException; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.extensions.restapi.RestModifyView; -import com.google.gerrit.reviewdb.client.SubmoduleSubscription; -import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.IdentifiedUser; import com.google.gerrit.server.extensions.events.GitReferenceUpdated; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.project.DeleteBranch.Input; import com.google.gerrit.server.query.change.InternalChangeQuery; import com.google.gwtorm.server.OrmException; -import com.google.gwtorm.server.ResultSet; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; @@ -52,19 +49,17 @@ public class DeleteBranch implements RestModifyView{ private final Provider identifiedUser; private final GitRepositoryManager repoManager; - private final Provider dbProvider; private final Provider queryProvider; private final GitReferenceUpdated referenceUpdated; private final ChangeHooks hooks; @Inject DeleteBranch(Provider identifiedUser, - GitRepositoryManager repoManager, Provider dbProvider, + GitRepositoryManager repoManager, Provider queryProvider, GitReferenceUpdated referenceUpdated, ChangeHooks hooks) { this.identifiedUser = identifiedUser; this.repoManager = repoManager; - this.dbProvider = dbProvider; this.queryProvider = queryProvider; this.referenceUpdated = referenceUpdated; this.hooks = hooks; @@ -115,9 +110,6 @@ public class DeleteBranch implements RestModifyView{ case FORCED: referenceUpdated.fire(rsrc.getNameKey(), u, ReceiveCommand.Type.DELETE); hooks.doRefUpdatedHook(rsrc.getBranchKey(), u, identifiedUser.get().getAccount()); - ResultSet submoduleSubscriptions = - dbProvider.get().submoduleSubscriptions().bySuperProject(rsrc.getBranchKey()); - dbProvider.get().submoduleSubscriptions().delete(submoduleSubscriptions); break; case REJECTED_CURRENT_BRANCH: diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/project/DeleteBranches.java b/gerrit-server/src/main/java/com/google/gerrit/server/project/DeleteBranches.java index 7ec0a6f5fa..b851f9e1a6 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/project/DeleteBranches.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/project/DeleteBranches.java @@ -22,15 +22,12 @@ import com.google.gerrit.extensions.restapi.ResourceConflictException; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.extensions.restapi.RestModifyView; import com.google.gerrit.reviewdb.client.Branch; -import com.google.gerrit.reviewdb.client.SubmoduleSubscription; -import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.IdentifiedUser; import com.google.gerrit.server.extensions.events.GitReferenceUpdated; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.project.DeleteBranches.Input; import com.google.gerrit.server.query.change.InternalChangeQuery; import com.google.gwtorm.server.OrmException; -import com.google.gwtorm.server.ResultSet; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; @@ -69,7 +66,6 @@ class DeleteBranches implements RestModifyView { private final Provider identifiedUser; private final GitRepositoryManager repoManager; - private final Provider dbProvider; private final Provider queryProvider; private final GitReferenceUpdated referenceUpdated; private final ChangeHooks hooks; @@ -77,13 +73,11 @@ class DeleteBranches implements RestModifyView { @Inject DeleteBranches(Provider identifiedUser, GitRepositoryManager repoManager, - Provider dbProvider, Provider queryProvider, GitReferenceUpdated referenceUpdated, ChangeHooks hooks) { this.identifiedUser = identifiedUser; this.repoManager = repoManager; - this.dbProvider = dbProvider; this.queryProvider = queryProvider; this.referenceUpdated = referenceUpdated; this.hooks = hooks; @@ -167,15 +161,11 @@ class DeleteBranches implements RestModifyView { errorMessages.append("\n"); } - private void postDeletion(ProjectResource project, ReceiveCommand cmd) - throws OrmException { + private void postDeletion(ProjectResource project, ReceiveCommand cmd) { referenceUpdated.fire(project.getNameKey(), cmd); Branch.NameKey branchKey = new Branch.NameKey(project.getNameKey(), cmd.getRefName()); hooks.doRefUpdatedHook(branchKey, cmd.getOldId(), cmd.getNewId(), identifiedUser.get().getAccount()); - ResultSet submoduleSubscriptions = - dbProvider.get().submoduleSubscriptions().bySuperProject(branchKey); - dbProvider.get().submoduleSubscriptions().delete(submoduleSubscriptions); } } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaVersion.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaVersion.java index 310cae88a6..edeb583891 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaVersion.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/SchemaVersion.java @@ -32,7 +32,7 @@ import java.util.List; /** A version of the database schema. */ public abstract class SchemaVersion { /** The current schema version. */ - public static final Class C = Schema_119.class; + public static final Class C = Schema_120.class; public static int getBinaryVersion() { return guessVersion(C); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/schema/Schema_120.java b/gerrit-server/src/main/java/com/google/gerrit/server/schema/Schema_120.java new file mode 100644 index 0000000000..d0c03617b2 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/schema/Schema_120.java @@ -0,0 +1,114 @@ +// Copyright (C) 2016 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.schema; + +import com.google.gerrit.common.data.SubscribeSection; +import com.google.gerrit.reviewdb.client.Branch; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.reviewdb.server.ReviewDb; +import com.google.gerrit.server.extensions.events.GitReferenceUpdated; +import com.google.gerrit.server.git.GitRepositoryManager; +import com.google.gerrit.server.git.MetaDataUpdate; +import com.google.gerrit.server.git.ProjectConfig; +import com.google.gwtorm.jdbc.JdbcSchema; +import com.google.gwtorm.server.OrmException; +import com.google.inject.Inject; +import com.google.inject.Provider; + +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.BatchRefUpdate; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.RefSpec; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +public class Schema_120 extends SchemaVersion { + + private final GitRepositoryManager mgr; + + @Inject + Schema_120(Provider prior, + GitRepositoryManager mgr) { + super(prior); + this.mgr = mgr; + } + + private void allowSubmoduleSubscription(Branch.NameKey subbranch, + Branch.NameKey superBranch) throws OrmException { + try (Repository git = mgr.openRepository(subbranch.getParentKey()); + RevWalk rw = new RevWalk(git)) { + BatchRefUpdate bru = git.getRefDatabase().newBatchUpdate(); + try(MetaDataUpdate md = new MetaDataUpdate(GitReferenceUpdated.DISABLED, + subbranch.getParentKey(), git, bru)) { + md.setMessage("Added superproject subscription during upgrade"); + ProjectConfig pc = ProjectConfig.read(md); + + SubscribeSection s = null; + for (SubscribeSection s1 : pc.getSubscribeSections(subbranch)) { + if (s.getProject() == superBranch.getParentKey()) { + s = s1; + } + } + if (s == null) { + s = new SubscribeSection(superBranch.getParentKey()); + pc.addSubscribeSection(s); + } + RefSpec newRefSpec = new RefSpec(subbranch.get() + ":" + superBranch.get()); + + if (!s.getRefSpecs().contains(newRefSpec)) { + // For the migration we use only exact RefSpecs, we're not trying to + // generalize it. + s.addRefSpec(newRefSpec); + } + + pc.commit(md); + } + bru.execute(rw, NullProgressMonitor.INSTANCE); + } catch (ConfigInvalidException | IOException e) { + throw new OrmException(e); + } + } + + @Override + protected void migrateData(ReviewDb db, UpdateUI ui) + throws OrmException, SQLException { + ui.message("Generating Superproject subscriptions table to submodule ACLs"); + + try (Statement stmt = ((JdbcSchema) db).getConnection().createStatement(); + ResultSet rs = stmt.executeQuery("SELECT " + + "key.super_project.project_name, " + + "key.super_project.branch_name, " + + "submodule.project_name " + + "submodule.branch_name " + + "FROM submodule_subscriptions");) { + while (rs.next()) { + Project.NameKey superproject = new Project.NameKey(rs.getString(1)); + Branch.NameKey superbranch = new Branch.NameKey(superproject, + rs.getString(2)); + + Project.NameKey submodule = new Project.NameKey(rs.getString(4)); + Branch.NameKey subbranch = new Branch.NameKey(submodule, + rs.getString(5)); + + allowSubmoduleSubscription(subbranch, superbranch); + } + } + } +} diff --git a/gerrit-server/src/test/java/com/google/gerrit/testutil/DisabledReviewDb.java b/gerrit-server/src/test/java/com/google/gerrit/testutil/DisabledReviewDb.java index c526cea7d6..fa7a5c4ccf 100644 --- a/gerrit-server/src/test/java/com/google/gerrit/testutil/DisabledReviewDb.java +++ b/gerrit-server/src/test/java/com/google/gerrit/testutil/DisabledReviewDb.java @@ -33,7 +33,6 @@ import com.google.gerrit.reviewdb.server.PatchSetApprovalAccess; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.reviewdb.server.SchemaVersionAccess; import com.google.gerrit.reviewdb.server.StarredChangeAccess; -import com.google.gerrit.reviewdb.server.SubmoduleSubscriptionAccess; import com.google.gerrit.reviewdb.server.SystemConfigAccess; import com.google.gwtorm.server.Access; import com.google.gwtorm.server.StatementExecutor; @@ -163,11 +162,6 @@ public class DisabledReviewDb implements ReviewDb { throw new Disabled(); } - @Override - public SubmoduleSubscriptionAccess submoduleSubscriptions() { - throw new Disabled(); - } - @Override public AccountGroupByIdAccess accountGroupById() { throw new Disabled();