diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index 881f19d917..acf68cff88 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt @@ -1343,6 +1343,68 @@ screen: If `download.archive` is not specified defaults to all archive commands. Set to `off` or empty string to disable. +[[gc]] +=== Section gc + +This section allows to configure the git garbage collection and schedules it +to run periodically. It will be triggered and executed sequentially for all +projects. + +[[gc.startTime]]gc.startTime:: ++ +Start time to define the first execution of the git garbage collection. +If the configured `'gc.interval'` is shorter than `'gc.startTime - now'` +the start time will be preponed by the maximum integral multiple of +`'gc.interval'` so that the start time is still in the future. ++ +---- + : +or +: + + : Mon, Tue, Wed, Thu, Fri, Sat, Sun + : 00-23 + : 0-59 +---- + + +[[gc.interval]]gc.interval:: ++ +Interval for periodic repetition of triggering the git garbage collection. +The interval must be larger than zero. The following suffixes are supported +to define the time unit for the interval: ++ +* `s, sec, second, seconds` +* `m, min, minute, minutes` +* `h, hr, hour, hours` +* `d, day, days` +* `w, week, weeks` (`1 week` is treated as `7 days`) +* `mon, month, months` (`1 month` is treated as `30 days`) +* `y, year, years` (`1 year` is treated as `365 days`) + +Examples:: ++ +---- +gc.startTime = Fri 10:30 +gc.interval = 2 day +---- ++ +Assuming the server is started on Mon 7:00 -> `'startTime - now = 4 days 3:30 hours'`. +This is larger than the interval hence prepone the start time +by the maximum integral multiple of the interval so that start +time is still in the future, i.e. prepone by 4 days. This yields +a start time of Mon 10:30, next executions are Wed 10:30, Fri 10:30 +etc. ++ +---- +gc.startTime = 6:00 +gc.interval = 1 day +---- ++ +Assuming the server is started on Mon 7:00 this yields the first run on next Tuesday +at 6:00 and a repetition interval of 1 day. + + [[gerrit]] === Section gerrit diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java index 0ba0ce2b96..d0624d3a07 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java @@ -53,6 +53,7 @@ import com.google.gerrit.server.config.GerritGlobalModule; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.MasterNodeStartup; import com.google.gerrit.server.contact.HttpContactStoreConnection; +import com.google.gerrit.server.git.GarbageCollectionRunner; import com.google.gerrit.server.git.ReceiveCommitsExecutorModule; import com.google.gerrit.server.git.WorkQueue; import com.google.gerrit.server.index.IndexModule; @@ -353,6 +354,7 @@ public class Daemon extends SiteProgram { bind(GerritUiOptions.class).toInstance(new GerritUiOptions(headless)); } }); + modules.add(GarbageCollectionRunner.module()); return cfgInjector.createChildInjector(modules); } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/config/GcConfig.java b/gerrit-server/src/main/java/com/google/gerrit/server/config/GcConfig.java new file mode 100644 index 0000000000..54096bb086 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/config/GcConfig.java @@ -0,0 +1,36 @@ +// Copyright (C) 2014 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.config; + +import com.google.inject.Inject; +import com.google.inject.Singleton; + +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; + +@Singleton +public class GcConfig { + private final ScheduleConfig scheduleConfig; + + @Inject + GcConfig(@GerritServerConfig Config cfg) { + scheduleConfig = new ScheduleConfig(cfg, ConfigConstants.CONFIG_GC_SECTION); + } + + public ScheduleConfig getScheduleConfig() { + return scheduleConfig; + } + +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java b/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java index 023c11521e..99ad937f9e 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java @@ -218,6 +218,8 @@ public class GerritGlobalModule extends FactoryModule { bind(EventFactory.class); bind(TransferConfig.class); + bind(GcConfig.class); + bind(ApprovalsUtil.class); bind(ChangeMergeQueue.class).in(SINGLETON); bind(MergeQueue.class).to(ChangeMergeQueue.class).in(SINGLETON); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/config/ScheduleConfig.java b/gerrit-server/src/main/java/com/google/gerrit/server/config/ScheduleConfig.java new file mode 100644 index 0000000000..bb8602ee8c --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/config/ScheduleConfig.java @@ -0,0 +1,128 @@ +// Copyright (C) 2014 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.config; + +import org.eclipse.jgit.lib.Config; +import org.joda.time.DateTime; +import org.joda.time.LocalDateTime; +import org.joda.time.LocalTime; +import org.joda.time.MutableDateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.MessageFormat; +import java.util.concurrent.TimeUnit; + +public class ScheduleConfig { + private static final Logger log = LoggerFactory + .getLogger(ScheduleConfig.class); + public static final long MISSING_CONFIG = -1L; + public static final long INVALID_CONFIG = -2L; + private static final String KEY_INTERVAL = "interval"; + private static final String KEY_STARTTIME = "startTime"; + + private final long initialDelay; + private final long interval; + + public ScheduleConfig(Config rc, String section) { + this(rc, section, null); + } + + public ScheduleConfig(Config rc, String section, String subsection) { + this(rc, section, subsection, DateTime.now()); + } + + /* For testing we need to be able to pass now */ + ScheduleConfig(Config rc, String section, String subsection, DateTime now) { + this.interval = interval(rc, section, subsection); + this.initialDelay = initialDelay(rc, section, subsection, now, interval); + } + + public long getInitialDelay() { + return initialDelay; + } + + public long getInterval() { + return interval; + } + + private static long interval(Config rc, String section, String subsection) { + long interval = MISSING_CONFIG; + try { + interval = + ConfigUtil.getTimeUnit(rc, section, subsection, KEY_INTERVAL, -1, + TimeUnit.MILLISECONDS); + if (interval == MISSING_CONFIG) { + log.info(MessageFormat.format( + "{0} schedule parameter \"{0}.{1}\" is not configured", section, + KEY_INTERVAL)); + } + } catch (IllegalArgumentException e) { + log.error(MessageFormat.format( + "Invalid {0} schedule parameter \"{0}.{1}\"", section, KEY_INTERVAL), + e); + interval = INVALID_CONFIG; + } + return interval; + } + + private static long initialDelay(Config rc, String section, + String subsection, DateTime now, long interval) { + long delay = MISSING_CONFIG; + String start = rc.getString(section, subsection, KEY_STARTTIME); + try { + if (start != null) { + DateTimeFormatter formatter = ISODateTimeFormat.hourMinute(); + MutableDateTime startTime = now.toMutableDateTime(); + LocalTime firstStartTime = null; + LocalDateTime firstStartDateTime = null; + try { + firstStartTime = formatter.parseLocalTime(start); + startTime.hourOfDay().set(firstStartTime.getHourOfDay()); + startTime.minuteOfHour().set(firstStartTime.getMinuteOfHour()); + } catch (IllegalArgumentException e1) { + formatter = DateTimeFormat.forPattern("E HH:mm"); + firstStartDateTime = formatter.parseLocalDateTime(start); + startTime.dayOfWeek().set(firstStartDateTime.getDayOfWeek()); + startTime.hourOfDay().set(firstStartDateTime.getHourOfDay()); + startTime.minuteOfHour().set(firstStartDateTime.getMinuteOfHour()); + } + startTime.secondOfMinute().set(0); + startTime.millisOfSecond().set(0); + while (startTime.isBefore(now)) { + startTime.add(interval); + } + while (startTime.getMillis() - now.getMillis() > interval) { + startTime.add(-interval); + } + delay = startTime.getMillis() - now.getMillis(); + } else { + log.info(MessageFormat.format( + "{0} schedule parameter \"{0}.{1}\" is not configured", section, + KEY_STARTTIME)); + } + } catch (IllegalArgumentException e2) { + log.error( + MessageFormat.format("Invalid {0} schedule parameter \"{0}.{1}\"", + section, KEY_STARTTIME), e2); + delay = INVALID_CONFIG; + } + return delay; + } + +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollectionRunner.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollectionRunner.java new file mode 100644 index 0000000000..10f2c8bac2 --- /dev/null +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollectionRunner.java @@ -0,0 +1,103 @@ +// Copyright (C) 2014 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.git; + +import static com.google.gerrit.server.config.ScheduleConfig.MISSING_CONFIG; + +import com.google.common.collect.Lists; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.gerrit.lifecycle.LifecycleModule; +import com.google.gerrit.server.config.GcConfig; +import com.google.gerrit.server.config.ScheduleConfig; +import com.google.gerrit.server.project.ProjectCache; +import com.google.inject.Inject; +import com.google.inject.Module; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; + +/** Runnable to enable scheduling gc to run periodically */ +public class GarbageCollectionRunner implements Runnable { + public static final String LOG_NAME = "gc_log"; + private static final Logger gcLog = LoggerFactory.getLogger(LOG_NAME); + + public static Module module() { + return new LifecycleModule() { + + @Override + protected void configure() { + listener().to(Lifecycle.class); + } + }; + } + + static class Lifecycle implements LifecycleListener { + private final WorkQueue queue; + private final GarbageCollectionRunner gcRunner; + private final GcConfig gcConfig; + + @Inject + Lifecycle(WorkQueue queue, GarbageCollectionRunner gcRunner, + GcConfig config) { + this.queue = queue; + this.gcRunner = gcRunner; + this.gcConfig = config; + } + + @Override + public void start() { + ScheduleConfig scheduleConfig = gcConfig.getScheduleConfig(); + long interval = scheduleConfig.getInterval(); + long delay = scheduleConfig.getInitialDelay(); + if (delay == MISSING_CONFIG && interval == MISSING_CONFIG) { + gcLog.info("Ignoring missing gc schedule configuration"); + } else if (delay < 0 || interval <= 0) { + gcLog.warn("Ignoring invalid gc schedule configuration"); + } else { + queue.getDefaultQueue().scheduleWithFixedDelay(gcRunner, delay, + interval, TimeUnit.MILLISECONDS); + } + } + + @Override + public void stop() { + // handled by WorkQueue.stop() already + } + } + + private final GarbageCollection.Factory garbageCollectionFactory; + private final ProjectCache projectCache; + + @Inject + GarbageCollectionRunner(GarbageCollection.Factory garbageCollectionFactory, + ProjectCache projectCache) { + this.garbageCollectionFactory = garbageCollectionFactory; + this.projectCache = projectCache; + } + + @Override + public void run() { + gcLog.info("Triggering gc on all repositories"); + garbageCollectionFactory.create().run( + Lists.newArrayList(projectCache.all())); + } + + @Override + public String toString() { + return "GC runner"; + } +} diff --git a/gerrit-server/src/test/java/com/google/gerrit/server/config/ScheduleConfigTest.java b/gerrit-server/src/test/java/com/google/gerrit/server/config/ScheduleConfigTest.java new file mode 100644 index 0000000000..a1dc6238a8 --- /dev/null +++ b/gerrit-server/src/test/java/com/google/gerrit/server/config/ScheduleConfigTest.java @@ -0,0 +1,83 @@ +// Copyright (C) 2014 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.config; + +import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.junit.Assert.assertEquals; + +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.Config; +import org.joda.time.DateTime; +import org.junit.Test; + +import java.text.MessageFormat; +import java.util.concurrent.TimeUnit; + +public class ScheduleConfigTest { + + // Friday June 13, 2014 10:00 UTC + private static final DateTime NOW = DateTime.parse("2014-06-13T10:00:00-00:00"); + + @Test + public void testInitialDelay() throws Exception { + assertEquals(ms(1, HOURS), initialDelay("11:00", "1h")); + assertEquals(ms(30, MINUTES), initialDelay("05:30", "1h")); + assertEquals(ms(30, MINUTES), initialDelay("09:30", "1h")); + assertEquals(ms(30, MINUTES), initialDelay("13:30", "1h")); + assertEquals(ms(59, MINUTES), initialDelay("13:59", "1h")); + + assertEquals(ms(1, HOURS), initialDelay("11:00", "1d")); + assertEquals(ms(19, HOURS) + ms(30, MINUTES), initialDelay("05:30", "1d")); + + assertEquals(ms(1, HOURS), initialDelay("11:00", "1w")); + assertEquals(ms(7, DAYS) - ms(4, HOURS) - ms(30, MINUTES), + initialDelay("05:30", "1w")); + + assertEquals(ms(3, DAYS) + ms(1, HOURS), initialDelay("Mon 11:00", "1w")); + assertEquals(ms(1, HOURS), initialDelay("Fri 11:00", "1w")); + + assertEquals(ms(1, HOURS), initialDelay("Mon 11:00", "1d")); + assertEquals(ms(23, HOURS), initialDelay("Mon 09:00", "1d")); + assertEquals(0, initialDelay("Mon 10:00", "1d")); + } + + private static long initialDelay(String startTime, String interval) + throws ConfigInvalidException { + return config(startTime, interval).getInitialDelay(); + } + + private static ScheduleConfig config(String startTime, String interval) + throws ConfigInvalidException { + Config rc = + readConfig(MessageFormat.format( + "[section \"subsection\"]\nstartTime = {0}\ninterval = {1}\n", + startTime, interval)); + return new ScheduleConfig(rc, "section", "subsection", NOW); + } + + private static Config readConfig(String dat) + throws ConfigInvalidException { + Config config = new Config(); + config.fromText(dat); + return config; + } + + private static long ms(int cnt, TimeUnit unit) { + return MILLISECONDS.convert(cnt, unit); + } +} diff --git a/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java b/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java index a0f4c36300..9e5ffbbb9c 100644 --- a/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java +++ b/gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java @@ -37,6 +37,7 @@ import com.google.gerrit.server.config.GerritServerConfigModule; import com.google.gerrit.server.config.MasterNodeStartup; import com.google.gerrit.server.config.SitePath; import com.google.gerrit.server.contact.HttpContactStoreConnection; +import com.google.gerrit.server.git.GarbageCollectionRunner; import com.google.gerrit.server.git.LocalDiskRepositoryManager; import com.google.gerrit.server.git.ReceiveCommitsExecutorModule; import com.google.gerrit.server.git.WorkQueue; @@ -310,6 +311,7 @@ public class WebAppInitializer extends GuiceServletContextListener bind(GerritUiOptions.class).toInstance(new GerritUiOptions(false)); } }); + modules.add(GarbageCollectionRunner.module()); return cfgInjector.createChildInjector(modules); }