diff --git a/java/com/google/gerrit/acceptance/InProcessProtocol.java b/java/com/google/gerrit/acceptance/InProcessProtocol.java index 7a79ce4929..30845a83d6 100644 --- a/java/com/google/gerrit/acceptance/InProcessProtocol.java +++ b/java/com/google/gerrit/acceptance/InProcessProtocol.java @@ -14,6 +14,10 @@ package com.google.gerrit.acceptance; +import static com.google.gerrit.server.git.receive.LazyPostReceiveHookChain.affectsSize; +import static com.google.gerrit.server.quota.QuotaGroupDefinitions.REPOSITORY_SIZE_GROUP; + +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.gerrit.acceptance.InProcessProtocol.Context; import com.google.gerrit.common.data.Capable; @@ -40,6 +44,9 @@ import com.google.gerrit.server.permissions.ProjectPermission; import com.google.gerrit.server.plugincontext.PluginSetContext; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.project.ProjectState; +import com.google.gerrit.server.quota.QuotaBackend; +import com.google.gerrit.server.quota.QuotaException; +import com.google.gerrit.server.quota.QuotaResponse; import com.google.gerrit.server.util.RequestContext; import com.google.gerrit.server.util.RequestScopePropagator; import com.google.gerrit.server.util.ThreadLocalRequestContext; @@ -263,6 +270,7 @@ class InProcessProtocol extends TestProtocol { private final DynamicSet postReceiveHooks; private final ThreadLocalRequestContext threadContext; private final PermissionBackend permissionBackend; + private final QuotaBackend quotaBackend; @Inject Receive( @@ -273,7 +281,8 @@ class InProcessProtocol extends TestProtocol { PluginSetContext receivePackInitializers, DynamicSet postReceiveHooks, ThreadLocalRequestContext threadContext, - PermissionBackend permissionBackend) { + PermissionBackend permissionBackend, + QuotaBackend quotaBackend) { this.userProvider = userProvider; this.projectCache = projectCache; this.factory = factory; @@ -282,6 +291,7 @@ class InProcessProtocol extends TestProtocol { this.postReceiveHooks = postReceiveHooks; this.threadContext = threadContext; this.permissionBackend = permissionBackend; + this.quotaBackend = quotaBackend; } @Override @@ -321,10 +331,35 @@ class InProcessProtocol extends TestProtocol { receivePackInitializers.runEach( initializer -> initializer.init(projectState.getNameKey(), rp)); + QuotaResponse.Aggregated availableTokens = + quotaBackend + .user(identifiedUser) + .project(req.project) + .availableTokens(REPOSITORY_SIZE_GROUP); + availableTokens.throwOnError(); + availableTokens.availableTokens().ifPresent(v -> rp.setMaxObjectSizeLimit(v)); - rp.setPostReceiveHook(PostReceiveHookChain.newChain(Lists.newArrayList(postReceiveHooks))); + ImmutableList hooks = + ImmutableList.builder() + .add( + (pack, commands) -> { + if (affectsSize(pack, commands)) { + try { + quotaBackend + .user(identifiedUser) + .project(req.project) + .requestTokens(REPOSITORY_SIZE_GROUP, pack.getPackSize()) + .throwOnError(); + } catch (QuotaException e) { + throw new RuntimeException(e); + } + } + }) + .addAll(postReceiveHooks) + .build(); + rp.setPostReceiveHook(PostReceiveHookChain.newChain(hooks)); return rp; - } catch (IOException | PermissionBackendException e) { + } catch (IOException | PermissionBackendException | QuotaException e) { throw new RuntimeException(e); } } diff --git a/java/com/google/gerrit/server/git/receive/AsyncReceiveCommits.java b/java/com/google/gerrit/server/git/receive/AsyncReceiveCommits.java index 66e66ca859..da2887f907 100644 --- a/java/com/google/gerrit/server/git/receive/AsyncReceiveCommits.java +++ b/java/com/google/gerrit/server/git/receive/AsyncReceiveCommits.java @@ -14,6 +14,7 @@ package com.google.gerrit.server.git.receive; +import static com.google.gerrit.server.quota.QuotaGroupDefinitions.REPOSITORY_SIZE_GROUP; import static java.util.concurrent.TimeUnit.NANOSECONDS; import com.google.common.flogger.FluentLogger; @@ -46,6 +47,9 @@ import com.google.gerrit.server.permissions.ProjectPermission; import com.google.gerrit.server.project.ContributorAgreementsChecker; import com.google.gerrit.server.project.ProjectState; import com.google.gerrit.server.query.change.InternalChangeQuery; +import com.google.gerrit.server.quota.QuotaBackend; +import com.google.gerrit.server.quota.QuotaException; +import com.google.gerrit.server.quota.QuotaResponse; import com.google.gerrit.server.util.MagicBranch; import com.google.gerrit.server.util.RequestScopePropagator; import com.google.inject.Inject; @@ -96,6 +100,7 @@ public class AsyncReceiveCommits implements PreReceiveHook { public static class Module extends PrivateModule { @Override public void configure() { + install(new FactoryModuleBuilder().build(LazyPostReceiveHookChain.Factory.class)); install(new FactoryModuleBuilder().build(AsyncReceiveCommits.Factory.class)); expose(AsyncReceiveCommits.Factory.class); // Don't expose the binding for ReceiveCommits.Factory. All callers should @@ -253,9 +258,10 @@ public class AsyncReceiveCommits implements PreReceiveHook { RequestScopePropagator scopePropagator, ReceiveConfig receiveConfig, TransferConfig transferConfig, - Provider lazyPostReceive, + LazyPostReceiveHookChain.Factory lazyPostReceive, ContributorAgreementsChecker contributorAgreements, Metrics metrics, + QuotaBackend quotaBackend, @Named(TIMEOUT_NAME) long timeoutMillis, @Assisted ProjectState projectState, @Assisted IdentifiedUser user, @@ -284,7 +290,7 @@ public class AsyncReceiveCommits implements PreReceiveHook { receivePack.setRefFilter(new ReceiveRefFilter()); receivePack.setAllowPushOptions(true); receivePack.setPreReceiveHook(this); - receivePack.setPostReceiveHook(lazyPostReceive.get()); + receivePack.setPostReceiveHook(lazyPostReceive.create(user, projectName)); // If the user lacks READ permission, some references may be filtered and hidden from view. // Check objects mentioned inside the incoming pack file are reachable from visible refs. @@ -311,6 +317,17 @@ public class AsyncReceiveCommits implements PreReceiveHook { factory.create( projectState, user, receivePack, allRefsWatcher, messageSender, resultChangeIds); receiveCommits.init(); + QuotaResponse.Aggregated availableTokens = + quotaBackend.user(user).project(projectName).availableTokens(REPOSITORY_SIZE_GROUP); + try { + availableTokens.throwOnError(); + } catch (QuotaException e) { + logger.atWarning().withCause(e).log( + "Quota %s availableTokens request failed for project %s", + REPOSITORY_SIZE_GROUP, projectName); + throw new RuntimeException(e); + } + availableTokens.availableTokens().ifPresent(v -> receivePack.setMaxObjectSizeLimit(v)); } /** Determine if the user can upload commits. */ diff --git a/java/com/google/gerrit/server/git/receive/LazyPostReceiveHookChain.java b/java/com/google/gerrit/server/git/receive/LazyPostReceiveHookChain.java index 0f081bebb5..8e200eb67a 100644 --- a/java/com/google/gerrit/server/git/receive/LazyPostReceiveHookChain.java +++ b/java/com/google/gerrit/server/git/receive/LazyPostReceiveHookChain.java @@ -14,23 +14,78 @@ package com.google.gerrit.server.git.receive; +import static com.google.gerrit.server.quota.QuotaGroupDefinitions.REPOSITORY_SIZE_GROUP; + +import com.google.common.flogger.FluentLogger; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.plugincontext.PluginSetContext; +import com.google.gerrit.server.quota.QuotaBackend; +import com.google.gerrit.server.quota.QuotaResponse; import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; import java.util.Collection; import org.eclipse.jgit.transport.PostReceiveHook; import org.eclipse.jgit.transport.ReceiveCommand; import org.eclipse.jgit.transport.ReceivePack; -class LazyPostReceiveHookChain implements PostReceiveHook { +/** + * Class is responsible for calling all registered post-receive hooks. In addition, in case when + * repository size quota is defined, it requests tokens (pack size) that were received. This is the + * final step of enforcing repository size quota that deducts token from available tokens. + */ +public class LazyPostReceiveHookChain implements PostReceiveHook { + interface Factory { + LazyPostReceiveHookChain create(CurrentUser user, Project.NameKey project); + } + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private final PluginSetContext hooks; + private final QuotaBackend quotaBackend; + private final CurrentUser user; + private final Project.NameKey project; @Inject - LazyPostReceiveHookChain(PluginSetContext hooks) { + LazyPostReceiveHookChain( + PluginSetContext hooks, + QuotaBackend quotaBackend, + @Assisted CurrentUser user, + @Assisted Project.NameKey project) { this.hooks = hooks; + this.quotaBackend = quotaBackend; + this.user = user; + this.project = project; } @Override public void onPostReceive(ReceivePack rp, Collection commands) { hooks.runEach(h -> h.onPostReceive(rp, commands)); + if (affectsSize(rp, commands)) { + QuotaResponse.Aggregated a = + quotaBackend + .user(user) + .project(project) + .requestTokens(REPOSITORY_SIZE_GROUP, rp.getPackSize()); + if (a.hasError()) { + String msg = + String.format( + "%s request failed for project %s with [%s]", + REPOSITORY_SIZE_GROUP, project, a.errorMessage()); + logger.atWarning().log(msg); + throw new RuntimeException(msg); + } + } + } + + public static boolean affectsSize(ReceivePack rp, Collection commands) { + if (rp.getPackSize() > 0L) { + for (ReceiveCommand cmd : commands) { + if (cmd.getType() != ReceiveCommand.Type.DELETE) { + return true; + } + } + } + return false; } } diff --git a/java/com/google/gerrit/server/quota/QuotaGroupDefinitions.java b/java/com/google/gerrit/server/quota/QuotaGroupDefinitions.java new file mode 100644 index 0000000000..5110538866 --- /dev/null +++ b/java/com/google/gerrit/server/quota/QuotaGroupDefinitions.java @@ -0,0 +1,25 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.quota; + +public class QuotaGroupDefinitions { + /** + * Definition of repository size quota group. {@link QuotaEnforcer} implementations for repository + * size quota have to act on requests with this group name. + */ + public static final String REPOSITORY_SIZE_GROUP = "/repository:size"; + + private QuotaGroupDefinitions() {} +} diff --git a/javatests/com/google/gerrit/acceptance/server/quota/DefaultQuotaBackendIT.java b/javatests/com/google/gerrit/acceptance/server/quota/DefaultQuotaBackendIT.java index 3e902839b4..adc7807101 100644 --- a/javatests/com/google/gerrit/acceptance/server/quota/DefaultQuotaBackendIT.java +++ b/javatests/com/google/gerrit/acceptance/server/quota/DefaultQuotaBackendIT.java @@ -21,6 +21,8 @@ import static org.easymock.EasyMock.resetToStrict; import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.extensions.annotations.Exports; +import com.google.gerrit.extensions.common.ChangeInfo; +import com.google.gerrit.extensions.common.ChangeInput; import com.google.gerrit.extensions.config.FactoryModule; import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.server.IdentifiedUser; @@ -92,7 +94,7 @@ public class DefaultQuotaBackendIT extends AbstractDaemonTest { @Test public void requestTokenForUserAndChange() throws Exception { - Change.Id changeId = createChange().getChange().getId(); + Change.Id changeId = retrieveChangeId(); QuotaRequestContext ctx = QuotaRequestContext.builder() .user(identifiedAdmin) @@ -148,7 +150,7 @@ public class DefaultQuotaBackendIT extends AbstractDaemonTest { @Test public void availableTokensForUserAndChange() throws Exception { - Change.Id changeId = createChange().getChange().getId(); + Change.Id changeId = retrieveChangeId(); QuotaRequestContext ctx = QuotaRequestContext.builder() .user(identifiedAdmin) @@ -224,6 +226,13 @@ public class DefaultQuotaBackendIT extends AbstractDaemonTest { quotaBackend.user(identifiedAdmin).availableTokens("testGroup"); } + private Change.Id retrieveChangeId() throws Exception { + // use REST API so that repository size quota doesn't have to be stubbed + ChangeInfo changeInfo = + gApi.changes().create(new ChangeInput(project.get(), "master", "test")).get(); + return new Change.Id(changeInfo._number); + } + private static QuotaResponse.Aggregated singletonAggregation(QuotaResponse response) { return QuotaResponse.Aggregated.create(Collections.singleton(response)); } diff --git a/javatests/com/google/gerrit/acceptance/server/quota/RepositorySizeQuotaIT.java b/javatests/com/google/gerrit/acceptance/server/quota/RepositorySizeQuotaIT.java new file mode 100644 index 0000000000..0814230e30 --- /dev/null +++ b/javatests/com/google/gerrit/acceptance/server/quota/RepositorySizeQuotaIT.java @@ -0,0 +1,140 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.acceptance.server.quota; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assert_; +import static com.google.gerrit.server.quota.QuotaGroupDefinitions.REPOSITORY_SIZE_GROUP; +import static com.google.gerrit.server.quota.QuotaResponse.ok; +import static org.easymock.EasyMock.anyLong; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.resetToStrict; +import static org.easymock.EasyMock.verify; + +import com.google.gerrit.acceptance.AbstractDaemonTest; +import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.extensions.config.FactoryModule; +import com.google.gerrit.server.CurrentUser; +import com.google.gerrit.server.quota.QuotaBackend; +import com.google.gerrit.server.quota.QuotaResponse; +import com.google.inject.Module; +import java.util.Collections; +import org.easymock.EasyMock; +import org.eclipse.jgit.api.errors.TooLargeObjectInPackException; +import org.eclipse.jgit.api.errors.TransportException; +import org.junit.Before; +import org.junit.Test; + +@UseLocalDisk +public class RepositorySizeQuotaIT extends AbstractDaemonTest { + private static final QuotaBackend.WithResource quotaBackendWithResource = + EasyMock.createStrictMock(QuotaBackend.WithResource.class); + private static final QuotaBackend.WithUser quotaBackendWithUser = + EasyMock.createStrictMock(QuotaBackend.WithUser.class); + + @Override + public Module createModule() { + return new FactoryModule() { + @Override + public void configure() { + bind(QuotaBackend.class) + .toInstance( + new QuotaBackend() { + @Override + public WithUser currentUser() { + return quotaBackendWithUser; + } + + @Override + public WithUser user(CurrentUser user) { + return quotaBackendWithUser; + } + }); + } + }; + } + + @Before + public void setUp() { + resetToStrict(quotaBackendWithResource); + resetToStrict(quotaBackendWithUser); + } + + @Test + public void pushWithAvailableTokens() throws Exception { + expect(quotaBackendWithResource.availableTokens(REPOSITORY_SIZE_GROUP)) + .andReturn(singletonAggregation(ok(276L))) + .times(2); + expect(quotaBackendWithResource.requestTokens(eq(REPOSITORY_SIZE_GROUP), anyLong())) + .andReturn(singletonAggregation(ok())); + expect(quotaBackendWithUser.project(project)).andReturn(quotaBackendWithResource).anyTimes(); + replay(quotaBackendWithResource); + replay(quotaBackendWithUser); + pushCommit(); + verify(quotaBackendWithUser); + verify(quotaBackendWithResource); + } + + @Test + public void pushWithNotSufficientTokens() throws Exception { + long availableTokens = 1L; + expect(quotaBackendWithResource.availableTokens(REPOSITORY_SIZE_GROUP)) + .andReturn(singletonAggregation(ok(availableTokens))) + .anyTimes(); + expect(quotaBackendWithUser.project(project)).andReturn(quotaBackendWithResource).anyTimes(); + replay(quotaBackendWithResource); + replay(quotaBackendWithUser); + try { + pushCommit(); + assert_().fail("expected TooLargeObjectInPackException"); + } catch (TooLargeObjectInPackException e) { + String msg = e.getMessage(); + assertThat(msg).contains("Object too large"); + assertThat(msg) + .contains(String.format("Max object size limit is %d bytes.", availableTokens)); + } + verify(quotaBackendWithUser); + verify(quotaBackendWithResource); + } + + @Test + public void errorGettingAvailableTokens() throws Exception { + String msg = "quota error"; + expect(quotaBackendWithResource.availableTokens(REPOSITORY_SIZE_GROUP)) + .andReturn(singletonAggregation(QuotaResponse.error(msg))) + .anyTimes(); + expect(quotaBackendWithUser.project(project)).andReturn(quotaBackendWithResource).anyTimes(); + replay(quotaBackendWithResource); + replay(quotaBackendWithUser); + try { + pushCommit(); + assert_().fail("expected TransportException"); + } catch (TransportException e) { + // TransportException has not much info about the cause + } + verify(quotaBackendWithUser); + verify(quotaBackendWithResource); + } + + private void pushCommit() throws Exception { + createCommitAndPush(testRepo, "refs/heads/master", "test 01", "file.test", "some content"); + } + + private static QuotaResponse.Aggregated singletonAggregation(QuotaResponse response) { + return QuotaResponse.Aggregated.create(Collections.singleton(response)); + } +} diff --git a/javatests/com/google/gerrit/acceptance/server/quota/RestApiQuotaIT.java b/javatests/com/google/gerrit/acceptance/server/quota/RestApiQuotaIT.java index a07569044d..dc082feda9 100644 --- a/javatests/com/google/gerrit/acceptance/server/quota/RestApiQuotaIT.java +++ b/javatests/com/google/gerrit/acceptance/server/quota/RestApiQuotaIT.java @@ -20,6 +20,7 @@ import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; import com.google.gerrit.acceptance.AbstractDaemonTest; +import com.google.gerrit.extensions.common.ChangeInfo; import com.google.gerrit.extensions.common.ChangeInput; import com.google.gerrit.extensions.config.FactoryModule; import com.google.gerrit.reviewdb.client.Change; @@ -68,7 +69,7 @@ public class RestApiQuotaIT extends AbstractDaemonTest { @Test public void changeDetail() throws Exception { - Change.Id changeId = createChange().getChange().getId(); + Change.Id changeId = retrieveChangeId(); expect(quotaBackendWithResource.requestToken("/restapi/changes/detail:GET")) .andReturn(singletonAggregation(QuotaResponse.ok())); replay(quotaBackendWithResource); @@ -81,7 +82,7 @@ public class RestApiQuotaIT extends AbstractDaemonTest { @Test public void revisionDetail() throws Exception { - Change.Id changeId = createChange().getChange().getId(); + Change.Id changeId = retrieveChangeId(); expect(quotaBackendWithResource.requestToken("/restapi/changes/revisions/actions:GET")) .andReturn(singletonAggregation(QuotaResponse.ok())); replay(quotaBackendWithResource); @@ -130,6 +131,13 @@ public class RestApiQuotaIT extends AbstractDaemonTest { adminRestSession.get("/config/server/version").assertStatus(429); } + private Change.Id retrieveChangeId() throws Exception { + // use REST API so that repository size quota doesn't have to be stubbed + ChangeInfo changeInfo = + gApi.changes().create(new ChangeInput(project.get(), "master", "test")).get(); + return new Change.Id(changeInfo._number); + } + private static QuotaResponse.Aggregated singletonAggregation(QuotaResponse response) { return QuotaResponse.Aggregated.create(Collections.singleton(response)); }