From 619169b4ebec7a359565c6af6c052d74b09f2f86 Mon Sep 17 00:00:00 2001 From: Edwin Kempin Date: Thu, 9 Feb 2012 15:47:52 +0100 Subject: [PATCH] Add SSH command to run Git garbage collection Add a new SSH command that allows to run the Git garbage collection for specific or all Gerrit projects. The default parameters for the Git garbage collection can be defined in the user global Git configuration file of the system user that runs the Gerrit server. It is possible to specify repository specific parameters for the garbage collection in the Git configuration files on repository level. All GC runs are logged in a newly added GC log file. This log file also contains statistics of the repositories that were garbage collected. Change-Id: Ie8730e3db785d5e05d5b739cfb1ef87ba515e870 Signed-off-by: Edwin Kempin --- Documentation/access-control.txt | 8 + Documentation/cmd-gc.txt | 72 +++++++ Documentation/cmd-index.txt | 3 + Documentation/rest-api-accounts.txt | 4 + .../google/gerrit/acceptance/SshSession.java | 14 +- .../acceptance/ssh/GarbageCollectionIT.java | 167 ++++++++++++++++ .../common/data/GarbageCollectionResult.java | 82 ++++++++ .../gerrit/common/data/GlobalCapability.java | 4 + .../client/admin/AdminConstants.properties | 2 + .../java/com/google/gerrit/pgm/Daemon.java | 2 + .../main/java/com/google/gerrit/pgm/Init.java | 2 + .../pgm/util/GarbageCollectionLogFile.java | 140 +++++++++++++ .../gerrit/pgm/util/LogFileCompressor.java | 2 + .../server/account/CapabilityControl.java | 6 + .../server/account/GetCapabilities.java | 2 + .../server/config/GerritGlobalModule.java | 2 + .../gerrit/server/git/GarbageCollection.java | 184 ++++++++++++++++++ .../server/git/GarbageCollectionQueue.java | 42 ++++ .../sshd/commands/DefaultCommandModule.java | 1 + .../commands/GarbageCollectionCommand.java | 122 ++++++++++++ 20 files changed, 860 insertions(+), 1 deletion(-) create mode 100644 Documentation/cmd-gc.txt create mode 100644 gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/ssh/GarbageCollectionIT.java create mode 100644 gerrit-common/src/main/java/com/google/gerrit/common/data/GarbageCollectionResult.java create mode 100644 gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/GarbageCollectionLogFile.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollection.java create mode 100644 gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollectionQueue.java create mode 100644 gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/GarbageCollectionCommand.java diff --git a/Documentation/access-control.txt b/Documentation/access-control.txt index e2feb81a1a..ba64e931e0 100644 --- a/Documentation/access-control.txt +++ b/Documentation/access-control.txt @@ -1148,6 +1148,14 @@ Access Database Allow users to access the database using the `gsql` command. +[[capability_runGC]] +Run Garbage Collection +~~~~~~~~~~~~~~~~~~~~~~ + +Allow users to run the Git garbage collection for the repositories of +all projects. + + [[capability_startReplication]] Start Replication ~~~~~~~~~~~~~~~~~ diff --git a/Documentation/cmd-gc.txt b/Documentation/cmd-gc.txt new file mode 100644 index 0000000000..07b899ac74 --- /dev/null +++ b/Documentation/cmd-gc.txt @@ -0,0 +1,72 @@ +gerrit gc +========= + +NAME +---- +gerrit gc - Run the Git garbage collection + +SYNOPSIS +-------- +[verse] +'ssh' -p 'gerrit gc' + [--all] + ... + +DESCRIPTION +----------- +Runs the Git garbage collection for the specified projects. + +A Gerrit system administrator can define the default parameters that +should be used for running the garbage collection in the user global +Git configuration file of the system user that runs the Gerrit +server. + +Since the user global Git configuration file is overlaid with the Git +configuration on repository level it is possible to specify +repository specific parameters for the garbage collection in the Git +repository configuration of every project. + +ACCESS +------ +Caller must be a member of the privileged 'Administrators' group, +or have been granted the +link:access-control.html#capability_runGC[Run Garbage Collection] +global capability. + +SCRIPTING +--------- +This command is intended to be used in scripts. + +OPTIONS +------- +:: + Name of the projects for which the Git garbage collection should be run. + +--all:: + If specified the Git garbage collection is run for all projects + sequentially. + +EXAMPLES +-------- + +Run the Git garbage collection for the projects 'myProject' and +'yourProject': +===== + $ ssh -p 29418 review.example.com gerrit gc myProject yourProject + collecting garbage for "myProject": + ... + done. + + collecting garbage for "yourProject": + ... + done. +===== + +Run the Git garbage collection for all projects: +===== + $ ssh -p 29418 review.example.com gerrit gc --all +===== + +GERRIT +------ +Part of link:index.html[Gerrit Code Review] diff --git a/Documentation/cmd-index.txt b/Documentation/cmd-index.txt index c5462d0118..f82532d136 100644 --- a/Documentation/cmd-index.txt +++ b/Documentation/cmd-index.txt @@ -114,6 +114,9 @@ link:cmd-set-project.html[gerrit set-project]:: link:cmd-flush-caches.html[gerrit flush-caches]:: Flush some/all server caches from memory. +link:cmd-gc.html[gerrit gc]:: + Run the Git garbage collection. + link:cmd-gsql.html[gerrit gsql]:: Administrative interface to active database. diff --git a/Documentation/rest-api-accounts.txt b/Documentation/rest-api-accounts.txt index 277c1d92bc..b6cfa23b05 100644 --- a/Documentation/rest-api-accounts.txt +++ b/Documentation/rest-api-accounts.txt @@ -76,6 +76,7 @@ Administrator that has authenticated with digest authentication: "flushCaches": true, "viewConnections": true, "viewQueue": true, + "runGC": true, "startReplication": true } ---- @@ -313,6 +314,9 @@ link:access-control.html#capability_viewConnections[View Connections] capability. |`viewQueue` |not set if `false`|Whether the user has the link:access-control.html#capability_viewQueue[View Queue] capability. +|`runGC` |not set if `false`|Whether the user has the +link:access-control.html#capability_runGC[Run Garbage Collection] +capability. |`startReplication` |not set if `false`|Whether the user has the link:access-control.html#capability_startReplication[Start Replication] capability. diff --git a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/SshSession.java b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/SshSession.java index fe7c8ee69d..aae9236bcd 100644 --- a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/SshSession.java +++ b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/SshSession.java @@ -27,6 +27,7 @@ public class SshSession { private final TestAccount account; private Session session; + private String error; public SshSession(TestAccount account) { this.account = account; @@ -40,13 +41,24 @@ public class SshSession { InputStream in = channel.getInputStream(); channel.connect(); - Scanner s = new Scanner(in).useDelimiter("\\A"); + Scanner s = new Scanner(channel.getErrStream()).useDelimiter("\\A"); + error = s.hasNext() ? s.next() : null; + + s = new Scanner(in).useDelimiter("\\A"); return s.hasNext() ? s.next() : ""; } finally { channel.disconnect(); } } + public boolean hasError() { + return error != null; + } + + public String getError() { + return error; + } + public void close() { if (session != null) { session.disconnect(); diff --git a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/ssh/GarbageCollectionIT.java b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/ssh/GarbageCollectionIT.java new file mode 100644 index 0000000000..8c934fba0b --- /dev/null +++ b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/ssh/GarbageCollectionIT.java @@ -0,0 +1,167 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.acceptance.ssh; + +import static com.google.gerrit.acceptance.git.GitUtil.createProject; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.gerrit.acceptance.AbstractDaemonTest; +import com.google.gerrit.acceptance.AccountCreator; +import com.google.gerrit.acceptance.SshSession; +import com.google.gerrit.acceptance.TestAccount; +import com.google.gerrit.common.data.GarbageCollectionResult; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.config.AllProjectsName; +import com.google.gerrit.server.git.GarbageCollection; +import com.google.gerrit.server.git.GarbageCollectionQueue; +import com.google.gerrit.server.git.GitRepositoryManager; +import com.google.gwtorm.server.OrmException; +import com.google.inject.Inject; + +import com.jcraft.jsch.JSchException; + +import org.eclipse.jgit.errors.RepositoryNotFoundException; +import org.eclipse.jgit.lib.Repository; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Arrays; +import java.util.Locale; + +public class GarbageCollectionIT extends AbstractDaemonTest { + + @Inject + private AccountCreator accounts; + + @Inject + private GitRepositoryManager repoManager; + + @Inject + private AllProjectsName allProjects; + + @Inject + private GarbageCollection.Factory garbageCollectionFactory; + + @Inject + private GarbageCollectionQueue gcQueue; + + private TestAccount admin; + private SshSession sshSession; + private Project.NameKey project1; + private Project.NameKey project2; + private Project.NameKey project3; + + @Before + public void setUp() throws Exception { + admin = + accounts.create("admin", "admin@example.com", "Administrator", + "Administrators"); + + sshSession = new SshSession(admin); + + project1 = new Project.NameKey("p1"); + createProject(sshSession, project1.get()); + + project2 = new Project.NameKey("p2"); + createProject(sshSession, project2.get()); + + project3 = new Project.NameKey("p3"); + createProject(sshSession, project3.get()); + } + + @Test + public void testGc() throws JSchException, IOException { + String response = + sshSession.exec("gerrit gc \"" + project1.get() + "\" \"" + + project2.get() + "\""); + assertFalse(sshSession.hasError()); + assertNoError(response); + assertHasPackFile(project1, project2); + assertHasNoPackFile(allProjects, project3); + } + + @Test + public void testGcAll() throws JSchException, IOException { + String response = sshSession.exec("gerrit gc --all"); + assertFalse(sshSession.hasError()); + assertNoError(response); + assertHasPackFile(allProjects, project1, project2, project3); + } + + @Test + public void testGcWithoutCapability_Error() throws IOException, OrmException, + JSchException { + SshSession s = new SshSession(accounts.create("user", "user@example.com", "User")); + s.exec("gerrit gc --all"); + assertError("fatal: user does not have \"runGC\" capability.", s.getError()); + } + + @Test + public void testGcAlreadyScheduled() { + gcQueue.addAll(Arrays.asList(project1)); + GarbageCollectionResult result = garbageCollectionFactory.create().run( + Arrays.asList(allProjects, project1, project2, project3)); + assertTrue(result.hasErrors()); + assertEquals(1, result.getErrors().size()); + GarbageCollectionResult.Error error = result.getErrors().get(0); + assertEquals(GarbageCollectionResult.Error.Type.GC_ALREADY_SCHEDULED, error.getType()); + assertEquals(project1, error.getProjectName()); + } + + private void assertError(String expectedError, String response) { + assertTrue(response, response.contains(expectedError)); + } + + private void assertNoError(String response) { + assertFalse(response, response.toLowerCase(Locale.US).contains("error")); + } + + private void assertHasPackFile(Project.NameKey... projects) + throws RepositoryNotFoundException, IOException { + for (Project.NameKey p : projects) { + assertTrue("Project " + p.get() + "has no pack files.", + getPackFiles(p).length > 0); + } + } + + private void assertHasNoPackFile(Project.NameKey... projects) + throws RepositoryNotFoundException, IOException { + for (Project.NameKey p : projects) { + assertTrue("Project " + p.get() + "has pack files.", + getPackFiles(p).length == 0); + } + } + + private String[] getPackFiles(Project.NameKey p) + throws RepositoryNotFoundException, IOException { + Repository repo = repoManager.openRepository(p); + try { + File packDir = new File(repo.getDirectory(), "objects/pack"); + return packDir.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".pack"); + } + }); + } finally { + repo.close(); + } + } +} diff --git a/gerrit-common/src/main/java/com/google/gerrit/common/data/GarbageCollectionResult.java b/gerrit-common/src/main/java/com/google/gerrit/common/data/GarbageCollectionResult.java new file mode 100644 index 0000000000..93f283bebe --- /dev/null +++ b/gerrit-common/src/main/java/com/google/gerrit/common/data/GarbageCollectionResult.java @@ -0,0 +1,82 @@ +// Copyright (C) 2012 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.common.data; + +import com.google.common.collect.Lists; +import com.google.gerrit.reviewdb.client.Project; + +import java.util.List; + +public class GarbageCollectionResult { + protected List errors; + + public GarbageCollectionResult() { + errors = Lists.newArrayList(); + } + + public void addError(Error e) { + errors.add(e); + } + + public List getErrors() { + return errors; + } + + public boolean hasErrors() { + return !errors.isEmpty(); + } + + public static class Error { + public static enum Type { + /** Git garbage collection was already scheduled for this project */ + GC_ALREADY_SCHEDULED, + + /** The repository was not found. */ + REPOSITORY_NOT_FOUND, + + /** The Git garbage collection failed. */ + GC_FAILED + } + + protected Type type; + protected Project.NameKey projectName; + + protected Error() { + } + + public Error(Type type, Project.NameKey projectName) { + this.type = type; + this.projectName = projectName; + } + + public Type getType() { + return type; + } + + public Project.NameKey getProjectName() { + return projectName; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append(type); + if (projectName != null) { + b.append(" ").append(projectName); + } + return b.toString(); + } + } +} diff --git a/gerrit-common/src/main/java/com/google/gerrit/common/data/GlobalCapability.java b/gerrit-common/src/main/java/com/google/gerrit/common/data/GlobalCapability.java index 973cc6a90a..7db691d2d6 100644 --- a/gerrit-common/src/main/java/com/google/gerrit/common/data/GlobalCapability.java +++ b/gerrit-common/src/main/java/com/google/gerrit/common/data/GlobalCapability.java @@ -67,6 +67,9 @@ public class GlobalCapability { /** Maximum result limit per executed query. */ public static final String QUERY_LIMIT = "queryLimit"; + /** Can run the Git garbage collection. */ + public static final String RUN_GC = "runGC"; + /** Forcefully restart replication to any configured destination. */ public static final String START_REPLICATION = "startReplication"; @@ -94,6 +97,7 @@ public class GlobalCapability { NAMES_ALL.add(KILL_TASK); NAMES_ALL.add(PRIORITY); NAMES_ALL.add(QUERY_LIMIT); + NAMES_ALL.add(RUN_GC); NAMES_ALL.add(START_REPLICATION); NAMES_ALL.add(VIEW_CACHES); NAMES_ALL.add(VIEW_CONNECTIONS); diff --git a/gerrit-gwtui/src/main/java/com/google/gerrit/client/admin/AdminConstants.properties b/gerrit-gwtui/src/main/java/com/google/gerrit/client/admin/AdminConstants.properties index c8a7c6e902..1637919e0d 100644 --- a/gerrit-gwtui/src/main/java/com/google/gerrit/client/admin/AdminConstants.properties +++ b/gerrit-gwtui/src/main/java/com/google/gerrit/client/admin/AdminConstants.properties @@ -156,6 +156,7 @@ capabilityNames = \ killTask, \ priority, \ queryLimit, \ + runGC, \ startReplication, \ viewCaches, \ viewConnections, \ @@ -170,6 +171,7 @@ flushCaches = Flush Caches killTask = Kill Task priority = Priority queryLimit = Query Limit +runGC = Run Garbage Collection startReplication = Start Replication viewCaches = View Caches viewConnections = View Connections diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java index 9654804125..ca98a849d5 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java @@ -33,6 +33,7 @@ import com.google.gerrit.pgm.http.jetty.JettyEnv; import com.google.gerrit.pgm.http.jetty.JettyModule; import com.google.gerrit.pgm.http.jetty.ProjectQoSFilter; import com.google.gerrit.pgm.util.ErrorLogFile; +import com.google.gerrit.pgm.util.GarbageCollectionLogFile; import com.google.gerrit.pgm.util.LogFileCompressor; import com.google.gerrit.pgm.util.RuntimeShutdown; import com.google.gerrit.pgm.util.SiteProgram; @@ -163,6 +164,7 @@ public class Daemon extends SiteProgram { throw die("Cannot combine --slave and --enable-httpd"); } + manager.add(GarbageCollectionLogFile.start(getSitePath())); if (consoleLog) { } else { manager.add(ErrorLogFile.start(getSitePath())); diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Init.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Init.java index c0f0c4bb1f..b20133e969 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Init.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Init.java @@ -26,6 +26,7 @@ import com.google.gerrit.pgm.init.SitePathInitializer; import com.google.gerrit.pgm.util.ConsoleUI; import com.google.gerrit.pgm.util.Die; import com.google.gerrit.pgm.util.ErrorLogFile; +import com.google.gerrit.pgm.util.GarbageCollectionLogFile; import com.google.gerrit.pgm.util.IoUtil; import com.google.gerrit.pgm.util.SiteProgram; import com.google.gerrit.reviewdb.server.ReviewDb; @@ -66,6 +67,7 @@ public class Init extends SiteProgram { @Override public int run() throws Exception { ErrorLogFile.errorOnlyConsole(); + GarbageCollectionLogFile.errorOnlyConsole(); final SiteInit init = createSiteInit(); init.flags.autoStart = !noAutoStart && init.site.isNew; diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/GarbageCollectionLogFile.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/GarbageCollectionLogFile.java new file mode 100644 index 0000000000..ac5576fd73 --- /dev/null +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/GarbageCollectionLogFile.java @@ -0,0 +1,140 @@ +// Copyright (C) 2012 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.pgm.util; + +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.git.GarbageCollection; + +import org.apache.log4j.Appender; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.DailyRollingFileAppender; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.apache.log4j.helpers.OnlyOnceErrorHandler; +import org.apache.log4j.spi.ErrorHandler; +import org.apache.log4j.spi.LoggingEvent; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; + +public class GarbageCollectionLogFile { + private static final org.slf4j.Logger log = LoggerFactory.getLogger(GarbageCollectionLogFile.class); + + public static void errorOnlyConsole() { + LogManager.resetConfiguration(); + + PatternLayout layout = new PatternLayout(); + layout.setConversionPattern("%-5p %x: %m%n"); + + ConsoleAppender dst = new ConsoleAppender(); + dst.setLayout(layout); + dst.setTarget("System.err"); + dst.setThreshold(Level.ERROR); + + Logger root = LogManager.getLogger(GarbageCollection.LOG_NAME); + root.removeAllAppenders(); + root.addAppender(dst); + } + + public static LifecycleListener start(File sitePath) + throws FileNotFoundException { + File logdir = new SitePaths(sitePath).logs_dir; + if (!logdir.exists() && !logdir.mkdirs()) { + throw new Die("Cannot create log directory: " + logdir); + } + + PatternLayout layout = new PatternLayout(); + layout.setConversionPattern("[%d] %-5p %x: %m%n"); + + DailyRollingFileAppender dst = new DailyRollingFileAppender(); + dst.setName(GarbageCollection.LOG_NAME); + dst.setLayout(layout); + dst.setEncoding("UTF-8"); + dst.setFile(new File(resolve(logdir), GarbageCollection.LOG_NAME).getPath()); + dst.setImmediateFlush(true); + dst.setAppend(true); + dst.setThreshold(Level.INFO); + dst.setErrorHandler(new LogErrorHandler()); + dst.activateOptions(); + dst.setErrorHandler(new OnlyOnceErrorHandler()); + + Logger gcLogger = LogManager.getLogger(GarbageCollection.LOG_NAME); + gcLogger.removeAllAppenders(); + gcLogger.addAppender(dst); + gcLogger.setAdditivity(false); + + return new LifecycleListener() { + @Override + public void start() { + } + + @Override + public void stop() { + LogManager.getLogger(GarbageCollection.LOG_NAME).removeAllAppenders(); + } + }; + } + + private static File resolve(File logs_dir) { + try { + return logs_dir.getCanonicalFile(); + } catch (IOException e) { + return logs_dir.getAbsoluteFile(); + } + } + + private GarbageCollectionLogFile() { + } + + private static final class LogErrorHandler implements ErrorHandler { + @Override + public void error(String message, Exception e, int errorCode, + LoggingEvent event) { + error(e != null ? e.getMessage() : message); + } + + @Override + public void error(String message, Exception e, int errorCode) { + error(e != null ? e.getMessage() : message); + } + + @Override + public void error(String message) { + log.error("Cannot open '" + GarbageCollection.LOG_NAME + "' log file: " + + message); + } + + @Override + public void activateOptions() { + } + + @Override + public void setAppender(Appender appender) { + } + + @Override + public void setBackupAppender(Appender appender) { + } + + @Override + public void setLogger(Logger logger) { + } + } +} diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/LogFileCompressor.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/LogFileCompressor.java index 57cc7c4310..cda653e749 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/LogFileCompressor.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/LogFileCompressor.java @@ -19,6 +19,7 @@ import static java.util.concurrent.TimeUnit.HOURS; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.lifecycle.LifecycleModule; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.git.GarbageCollection; import com.google.gerrit.server.git.WorkQueue; import com.google.inject.Inject; @@ -97,6 +98,7 @@ public class LogFileCompressor implements Runnable { private boolean isLive(final File entry) { final String name = entry.getName(); return ErrorLogFile.LOG_NAME.equals(name) // + || GarbageCollection.LOG_NAME.equals(name) // || "sshd_log".equals(name) // || "httpd_log".equals(name) // || "gerrit.run".equals(name) // diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/account/CapabilityControl.java b/gerrit-server/src/main/java/com/google/gerrit/server/account/CapabilityControl.java index 706ae13a38..942b0d738f 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/account/CapabilityControl.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/account/CapabilityControl.java @@ -142,6 +142,12 @@ public class CapabilityControl { || canAdministrateServer(); } + /** @return true if the user can run the Git garbage collection. */ + public boolean canRunGC() { + return canPerform(GlobalCapability.RUN_GC) + || canAdministrateServer(); + } + /** @return which priority queue the user's tasks should be submitted to. */ public QueueProvider.QueueType getQueueType() { // If a non-generic group (that is not Anonymous Users or Registered Users) diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/account/GetCapabilities.java b/gerrit-server/src/main/java/com/google/gerrit/server/account/GetCapabilities.java index a432c6d447..81221aa50d 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/account/GetCapabilities.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/account/GetCapabilities.java @@ -22,6 +22,7 @@ import static com.google.gerrit.common.data.GlobalCapability.EMAIL_REVIEWERS; import static com.google.gerrit.common.data.GlobalCapability.FLUSH_CACHES; import static com.google.gerrit.common.data.GlobalCapability.KILL_TASK; import static com.google.gerrit.common.data.GlobalCapability.PRIORITY; +import static com.google.gerrit.common.data.GlobalCapability.RUN_GC; import static com.google.gerrit.common.data.GlobalCapability.START_REPLICATION; import static com.google.gerrit.common.data.GlobalCapability.VIEW_CACHES; import static com.google.gerrit.common.data.GlobalCapability.VIEW_CONNECTIONS; @@ -109,6 +110,7 @@ class GetCapabilities implements RestReadView { have.put(FLUSH_CACHES, cc.canFlushCaches()); have.put(VIEW_CONNECTIONS, cc.canViewConnections()); have.put(VIEW_QUEUE, cc.canViewQueue()); + have.put(RUN_GC, cc.canRunGC()); have.put(START_REPLICATION, cc.canStartReplication()); have.put(ACCESS_DATABASE, cc.canAccessDatabase()); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java index 297073f0ee..0c68c49557 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java @@ -67,6 +67,7 @@ import com.google.gerrit.server.events.EventFactory; import com.google.gerrit.server.extensions.events.GitReferenceUpdated; import com.google.gerrit.server.git.ChangeCache; import com.google.gerrit.server.git.ChangeMergeQueue; +import com.google.gerrit.server.git.GarbageCollection; import com.google.gerrit.server.git.GitModule; import com.google.gerrit.server.git.MergeQueue; import com.google.gerrit.server.git.MergeUtil; @@ -185,6 +186,7 @@ public class GerritGlobalModule extends FactoryModule { factory(RebasedPatchSetSender.Factory.class); factory(ReplacePatchSetSender.Factory.class); factory(PerformCreateProject.Factory.class); + factory(GarbageCollection.Factory.class); bind(PermissionCollection.Factory.class); bind(AccountVisibility.class) .toProvider(AccountVisibilityProvider.class) diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollection.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollection.java new file mode 100644 index 0000000000..685e87caa9 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollection.java @@ -0,0 +1,184 @@ +// Copyright (C) 2012 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.git; + +import com.google.common.collect.Sets; +import com.google.gerrit.common.data.GarbageCollectionResult; +import com.google.gerrit.reviewdb.client.Project; +import com.google.inject.Inject; + +import org.eclipse.jgit.api.GarbageCollectCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.errors.RepositoryNotFoundException; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.TextProgressMonitor; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.List; +import java.util.Properties; +import java.util.Set; + +public class GarbageCollection { + private static final Logger log = LoggerFactory + .getLogger(GarbageCollection.class); + + public static final String LOG_NAME = "gc_log"; + private static final Logger gcLog = LoggerFactory.getLogger(LOG_NAME); + + + private final GitRepositoryManager repoManager; + private final GarbageCollectionQueue gcQueue; + + public interface Factory { + GarbageCollection create(); + } + + @Inject + GarbageCollection(GitRepositoryManager repoManager, GarbageCollectionQueue gcQueue) { + this.repoManager = repoManager; + this.gcQueue = gcQueue; + } + + public GarbageCollectionResult run(List projectNames) { + return run(projectNames, null); + } + + public GarbageCollectionResult run(List projectNames, + PrintWriter writer) { + GarbageCollectionResult result = new GarbageCollectionResult(); + Set projectsToGc = gcQueue.addAll(projectNames); + for (Project.NameKey projectName : Sets.difference( + Sets.newHashSet(projectNames), projectsToGc)) { + result.addError(new GarbageCollectionResult.Error( + GarbageCollectionResult.Error.Type.GC_ALREADY_SCHEDULED, projectName)); + } + for (Project.NameKey p : projectsToGc) { + Repository repo = null; + try { + repo = repoManager.openRepository(p); + logGcConfiguration(p, repo); + print(writer, "collecting garbage for \"" + p + "\":\n"); + GarbageCollectCommand gc = Git.wrap(repo).gc(); + logGcInfo(p, "before:", gc.getStatistics()); + gc.setProgressMonitor(writer != null ? new TextProgressMonitor(writer) + : NullProgressMonitor.INSTANCE); + Properties statistics = gc.call(); + logGcInfo(p, "after: ", statistics); + print(writer, "done.\n\n"); + } catch (RepositoryNotFoundException e) { + logGcError(writer, p, e); + result.addError(new GarbageCollectionResult.Error( + GarbageCollectionResult.Error.Type.REPOSITORY_NOT_FOUND, + p)); + } catch (IOException e) { + logGcError(writer, p, e); + result.addError(new GarbageCollectionResult.Error( + GarbageCollectionResult.Error.Type.GC_FAILED, p)); + } catch (GitAPIException e) { + logGcError(writer, p, e); + result.addError(new GarbageCollectionResult.Error( + GarbageCollectionResult.Error.Type.GC_FAILED, p)); + } catch (JGitInternalException e) { + logGcError(writer, p, e); + result.addError(new GarbageCollectionResult.Error( + GarbageCollectionResult.Error.Type.GC_FAILED, p)); + } finally { + if (repo != null) { + repo.close(); + } + gcQueue.gcFinished(p); + } + } + return result; + } + + private static void logGcInfo(Project.NameKey projectName, String msg) { + logGcInfo(projectName, msg, null); + } + + private static void logGcInfo(Project.NameKey projectName, String msg, + Properties statistics) { + StringBuilder b = new StringBuilder(); + b.append("[").append(projectName.get()).append("] "); + b.append(msg); + if (statistics != null) { + b.append(" "); + String s = statistics.toString(); + if (s.startsWith("{") && s.endsWith("}")) { + s = s.substring(1, s.length() - 1); + } + b.append(s); + } + gcLog.info(b.toString()); + } + + private static void logGcConfiguration(Project.NameKey projectName, + Repository repo) { + StringBuilder b = new StringBuilder(); + Config cfg = repo.getConfig(); + b.append(formatConfigValues(cfg, ConfigConstants.CONFIG_GC_SECTION, null)); + for (String subsection : cfg.getSubsections(ConfigConstants.CONFIG_GC_SECTION)) { + b.append(formatConfigValues(cfg, ConfigConstants.CONFIG_GC_SECTION, + subsection)); + } + if (b.length() == 0) { + b.append("no set"); + } + + logGcInfo(projectName, "gc config: " + b.toString()); + logGcInfo(projectName, "pack config: " + (new PackConfig(repo)).toString()); + } + + private static String formatConfigValues(Config config, String section, + String subsection) { + StringBuilder b = new StringBuilder(); + Set names = config.getNames(section, subsection); + for (String name : names) { + String value = config.getString(section, subsection, name); + b.append(section); + if (subsection != null) { + b.append(".").append(subsection); + } + b.append("."); + b.append(name).append("=").append(value); + b.append("; "); + } + return b.toString(); + } + + private static void logGcError(PrintWriter writer, + Project.NameKey projectName, Exception e) { + print(writer, "failed.\n\n"); + StringBuilder b = new StringBuilder(); + b.append("[").append(projectName.get()).append("]"); + gcLog.error(b.toString(), e); + log.error(b.toString(), e); + } + + private static void print(PrintWriter writer, String message) { + if (writer != null) { + writer.print(message); + } + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollectionQueue.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollectionQueue.java new file mode 100644 index 0000000000..93d328a375 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollectionQueue.java @@ -0,0 +1,42 @@ +// Copyright (C) 2013 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.git; + +import com.google.common.collect.Sets; +import com.google.gerrit.reviewdb.client.Project; +import com.google.inject.Singleton; + +import java.util.Collection; +import java.util.Set; + +@Singleton +class GarbageCollectionQueue { + private final Set projectsScheduledForGc = Sets.newHashSet(); + + synchronized Set addAll(Collection projects) { + Set added = + Sets.newLinkedHashSetWithExpectedSize(projects.size()); + for (Project.NameKey p : projects) { + if (projectsScheduledForGc.add(p)) { + added.add(p); + } + } + return added; + } + + synchronized void gcFinished(Project.NameKey project) { + projectsScheduledForGc.remove(project); + } +} diff --git a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java index 2f89fcb5c2..35a4e141b8 100644 --- a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java +++ b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java @@ -47,6 +47,7 @@ public class DefaultCommandModule extends CommandModule { command(gerrit, ShowQueue.class); command(gerrit, StreamEvents.class); command(gerrit, VersionCommand.class); + command(gerrit, GarbageCollectionCommand.class); command(gerrit, "plugin").toProvider(new DispatchCommandProvider(plugin)); diff --git a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/GarbageCollectionCommand.java b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/GarbageCollectionCommand.java new file mode 100644 index 0000000000..c56115308f --- /dev/null +++ b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/GarbageCollectionCommand.java @@ -0,0 +1,122 @@ +// Copyright (C) 2012 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.sshd.commands; + +import com.google.common.collect.Lists; +import com.google.gerrit.common.data.GarbageCollectionResult; +import com.google.gerrit.common.data.GlobalCapability; +import com.google.gerrit.extensions.annotations.RequiresCapability; +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.git.GarbageCollection; +import com.google.gerrit.server.project.ProjectCache; +import com.google.gerrit.server.project.ProjectControl; +import com.google.gerrit.sshd.BaseCommand; +import com.google.gerrit.sshd.CommandMetaData; +import com.google.inject.Inject; + +import org.apache.sshd.server.Environment; +import org.kohsuke.args4j.Argument; +import org.kohsuke.args4j.Option; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +/** Runs the Git garbage collection. */ +@RequiresCapability(GlobalCapability.RUN_GC) +@CommandMetaData(name = "gc", descr = "Run Git garbage collection") +public class GarbageCollectionCommand extends BaseCommand { + + @Option(name = "--all", usage = "runs the Git garbage collection for all projects") + private boolean all; + + @Argument(index = 0, required = false, multiValued = true, metaVar = "NAME", + usage = "projects for which the Git garbage collection should be run") + private List projects = new ArrayList(); + + @Inject + private ProjectCache projectCache; + + @Inject + private GarbageCollection.Factory garbageCollectionFactory; + + private PrintWriter stdout; + + @Override + public void start(Environment env) throws IOException { + startThread(new CommandRunnable() { + @Override + public void run() throws Exception { + stdout = toPrintWriter(out); + try { + parseCommandLine(); + verifyCommandLine(); + runGC(); + } finally { + stdout.flush(); + } + } + }); + } + + private void verifyCommandLine() throws UnloggedFailure { + if (!all && projects.isEmpty()) { + throw new UnloggedFailure(1, + "needs projects as command arguments or --all option"); + } + if (all && !projects.isEmpty()) { + throw new UnloggedFailure(1, + "either specify projects as command arguments or use --all option"); + } + } + + private void runGC() { + List projectNames; + if (all) { + projectNames = Lists.newArrayList(projectCache.all()); + } else { + projectNames = Lists.newArrayListWithCapacity(projects.size()); + for (ProjectControl pc : projects) { + projectNames.add(pc.getProject().getNameKey()); + } + } + + GarbageCollectionResult result = + garbageCollectionFactory.create().run(projectNames, stdout); + if (result.hasErrors()) { + for (GarbageCollectionResult.Error e : result.getErrors()) { + String msg; + switch (e.getType()) { + case REPOSITORY_NOT_FOUND: + msg = "error: project \"" + e.getProjectName() + "\" not found"; + break; + case GC_ALREADY_SCHEDULED: + msg = "error: garbage collection for project \"" + + e.getProjectName() + "\" was already scheduled"; + break; + case GC_FAILED: + msg = "error: garbage collection for project \"" + e.getProjectName() + + "\" failed"; + break; + default: + msg = "error: garbage collection for project \"" + e.getProjectName() + + "\" failed: " + e.getType(); + } + stdout.print(msg + "\n"); + } + } + } +}