Allow to schedule a task to run gc for all projects

The (global) schedule can be configured in "gerrit.config" by specifying
startTime and interval. The "startTime" can be specified in section "gc"
in the following format:
startTime = <hours>:<minutes>
or
startTime = <day of week> <hours>:<minutes>
The "interval" can be given using the usual time unit suffixes.

Change-Id: I16e92c353d0fa05c020fc18da4ed1f5825ad677c

Documentation/config-gerrit.txt

@@ -1343,6 +1343,68 @@ screen:
 If `download.archive` is not specified defaults to all archive
 commands. Set to `off` or empty string to disable.
 
+[[gc]]
+=== Section gc
+
+This section allows to configure the git garbage collection and schedules it
+to run periodically. It will be triggered and executed sequentially for all
+projects.
+
+[[gc.startTime]]gc.startTime::
++
+Start time to define the first execution of the git garbage collection.
+If the configured `'gc.interval'` is shorter than `'gc.startTime - now'`
+the start time will be preponed by the maximum integral multiple of
+`'gc.interval'` so that the start time is still in the future.
++
+----
+<day of week> <hours>:<minutes>
+or
+<hours>:<minutes>
+
+<day of week> : Mon, Tue, Wed, Thu, Fri, Sat, Sun
+<hours>       : 00-23
+<minutes>     : 0-59
+----
+
+
+[[gc.interval]]gc.interval::
++
+Interval for periodic repetition of triggering the git garbage collection.
+The interval must be larger than zero. The following suffixes are supported
+to define the time unit for the interval:
++
+* `s, sec, second, seconds`
+* `m, min, minute, minutes`
+* `h, hr, hour, hours`
+* `d, day, days`
+* `w, week, weeks` (`1 week` is treated as `7 days`)
+* `mon, month, months` (`1 month` is treated as `30 days`)
+* `y, year, years` (`1 year` is treated as `365 days`)
+
+Examples::
++
+----
+gc.startTime = Fri 10:30
+gc.interval  = 2 day
+----
++
+Assuming the server is started on Mon 7:00 -> `'startTime - now = 4 days 3:30 hours'`.
+This is larger than the interval hence prepone the start time
+by the maximum integral multiple of the interval so that start
+time is still in the future, i.e. prepone by 4 days. This yields
+a start time of Mon 10:30, next executions are Wed 10:30, Fri 10:30
+etc.
++
+----
+gc.startTime = 6:00
+gc.interval = 1 day
+----
++
+Assuming the server is started on Mon 7:00 this yields the first run on next Tuesday
+at 6:00 and a repetition interval of 1 day.
+
+
 [[gerrit]]
 === Section gerrit
 

gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java

@@ -53,6 +53,7 @@ import com.google.gerrit.server.config.GerritGlobalModule;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.MasterNodeStartup;
 import com.google.gerrit.server.contact.HttpContactStoreConnection;
+import com.google.gerrit.server.git.GarbageCollectionRunner;
 import com.google.gerrit.server.git.ReceiveCommitsExecutorModule;
 import com.google.gerrit.server.git.WorkQueue;
 import com.google.gerrit.server.index.IndexModule;
@@ -353,6 +354,7 @@ public class Daemon extends SiteProgram {
         bind(GerritUiOptions.class).toInstance(new GerritUiOptions(headless));
       }
     });
+    modules.add(GarbageCollectionRunner.module());
     return cfgInjector.createChildInjector(modules);
   }
 

gerrit-server/src/main/java/com/google/gerrit/server/config/GcConfig.java

@@ -0,0 +1,36 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.config;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.ConfigConstants;
+
+@Singleton
+public class GcConfig {
+  private final ScheduleConfig scheduleConfig;
+
+  @Inject
+  GcConfig(@GerritServerConfig Config cfg) {
+    scheduleConfig = new ScheduleConfig(cfg, ConfigConstants.CONFIG_GC_SECTION);
+  }
+
+  public ScheduleConfig getScheduleConfig() {
+    return scheduleConfig;
+  }
+
+}

gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java

@@ -218,6 +218,8 @@ public class GerritGlobalModule extends FactoryModule {
     bind(EventFactory.class);
     bind(TransferConfig.class);
 
+    bind(GcConfig.class);
+
     bind(ApprovalsUtil.class);
     bind(ChangeMergeQueue.class).in(SINGLETON);
     bind(MergeQueue.class).to(ChangeMergeQueue.class).in(SINGLETON);

gerrit-server/src/main/java/com/google/gerrit/server/config/ScheduleConfig.java

@@ -0,0 +1,128 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.config;
+
+import org.eclipse.jgit.lib.Config;
+import org.joda.time.DateTime;
+import org.joda.time.LocalDateTime;
+import org.joda.time.LocalTime;
+import org.joda.time.MutableDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.ISODateTimeFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.MessageFormat;
+import java.util.concurrent.TimeUnit;
+
+public class ScheduleConfig {
+  private static final Logger log = LoggerFactory
+      .getLogger(ScheduleConfig.class);
+  public static final long MISSING_CONFIG = -1L;
+  public static final long INVALID_CONFIG = -2L;
+  private static final String KEY_INTERVAL = "interval";
+  private static final String KEY_STARTTIME = "startTime";
+
+  private final long initialDelay;
+  private final long interval;
+
+  public ScheduleConfig(Config rc, String section) {
+    this(rc, section, null);
+  }
+
+  public ScheduleConfig(Config rc, String section, String subsection) {
+    this(rc, section, subsection, DateTime.now());
+  }
+
+  /* For testing we need to be able to pass now */
+  ScheduleConfig(Config rc, String section, String subsection, DateTime now) {
+    this.interval = interval(rc, section, subsection);
+    this.initialDelay = initialDelay(rc, section, subsection, now, interval);
+  }
+
+  public long getInitialDelay() {
+    return initialDelay;
+  }
+
+  public long getInterval() {
+    return interval;
+  }
+
+  private static long interval(Config rc, String section, String subsection) {
+    long interval = MISSING_CONFIG;
+    try {
+      interval =
+          ConfigUtil.getTimeUnit(rc, section, subsection, KEY_INTERVAL, -1,
+              TimeUnit.MILLISECONDS);
+      if (interval == MISSING_CONFIG) {
+        log.info(MessageFormat.format(
+            "{0} schedule parameter \"{0}.{1}\" is not configured", section,
+            KEY_INTERVAL));
+      }
+    } catch (IllegalArgumentException e) {
+      log.error(MessageFormat.format(
+          "Invalid {0} schedule parameter \"{0}.{1}\"", section, KEY_INTERVAL),
+          e);
+      interval = INVALID_CONFIG;
+    }
+    return interval;
+  }
+
+  private static long initialDelay(Config rc, String section,
+      String subsection, DateTime now, long interval) {
+    long delay = MISSING_CONFIG;
+    String start = rc.getString(section, subsection, KEY_STARTTIME);
+    try {
+      if (start != null) {
+        DateTimeFormatter formatter = ISODateTimeFormat.hourMinute();
+        MutableDateTime startTime = now.toMutableDateTime();
+        LocalTime firstStartTime = null;
+        LocalDateTime firstStartDateTime = null;
+        try {
+          firstStartTime = formatter.parseLocalTime(start);
+          startTime.hourOfDay().set(firstStartTime.getHourOfDay());
+          startTime.minuteOfHour().set(firstStartTime.getMinuteOfHour());
+        } catch (IllegalArgumentException e1) {
+          formatter = DateTimeFormat.forPattern("E HH:mm");
+          firstStartDateTime = formatter.parseLocalDateTime(start);
+          startTime.dayOfWeek().set(firstStartDateTime.getDayOfWeek());
+          startTime.hourOfDay().set(firstStartDateTime.getHourOfDay());
+          startTime.minuteOfHour().set(firstStartDateTime.getMinuteOfHour());
+        }
+        startTime.secondOfMinute().set(0);
+        startTime.millisOfSecond().set(0);
+        while (startTime.isBefore(now)) {
+          startTime.add(interval);
+        }
+        while (startTime.getMillis() - now.getMillis() > interval) {
+          startTime.add(-interval);
+        }
+        delay = startTime.getMillis() - now.getMillis();
+      } else {
+        log.info(MessageFormat.format(
+            "{0} schedule parameter \"{0}.{1}\" is not configured", section,
+            KEY_STARTTIME));
+      }
+    } catch (IllegalArgumentException e2) {
+      log.error(
+          MessageFormat.format("Invalid {0} schedule parameter \"{0}.{1}\"",
+              section, KEY_STARTTIME), e2);
+      delay = INVALID_CONFIG;
+    }
+    return delay;
+  }
+
+}

gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollectionRunner.java

@@ -0,0 +1,103 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.git;
+
+import static com.google.gerrit.server.config.ScheduleConfig.MISSING_CONFIG;
+
+import com.google.common.collect.Lists;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.server.config.GcConfig;
+import com.google.gerrit.server.config.ScheduleConfig;
+import com.google.gerrit.server.project.ProjectCache;
+import com.google.inject.Inject;
+import com.google.inject.Module;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/** Runnable to enable scheduling gc to run periodically */
+public class GarbageCollectionRunner implements Runnable {
+  public static final String LOG_NAME = "gc_log";
+  private static final Logger gcLog = LoggerFactory.getLogger(LOG_NAME);
+
+  public static Module module() {
+    return new LifecycleModule() {
+
+      @Override
+      protected void configure() {
+        listener().to(Lifecycle.class);
+      }
+    };
+  }
+
+  static class Lifecycle implements LifecycleListener {
+    private final WorkQueue queue;
+    private final GarbageCollectionRunner gcRunner;
+    private final GcConfig gcConfig;
+
+    @Inject
+    Lifecycle(WorkQueue queue, GarbageCollectionRunner gcRunner,
+        GcConfig config) {
+      this.queue = queue;
+      this.gcRunner = gcRunner;
+      this.gcConfig = config;
+    }
+
+    @Override
+    public void start() {
+      ScheduleConfig scheduleConfig = gcConfig.getScheduleConfig();
+      long interval = scheduleConfig.getInterval();
+      long delay = scheduleConfig.getInitialDelay();
+      if (delay == MISSING_CONFIG && interval == MISSING_CONFIG) {
+        gcLog.info("Ignoring missing gc schedule configuration");
+      } else if (delay < 0 || interval <= 0) {
+        gcLog.warn("Ignoring invalid gc schedule configuration");
+      } else {
+        queue.getDefaultQueue().scheduleWithFixedDelay(gcRunner, delay,
+            interval, TimeUnit.MILLISECONDS);
+      }
+    }
+
+    @Override
+    public void stop() {
+      // handled by WorkQueue.stop() already
+    }
+  }
+
+  private final GarbageCollection.Factory garbageCollectionFactory;
+  private final ProjectCache projectCache;
+
+  @Inject
+  GarbageCollectionRunner(GarbageCollection.Factory garbageCollectionFactory,
+      ProjectCache projectCache) {
+    this.garbageCollectionFactory = garbageCollectionFactory;
+    this.projectCache = projectCache;
+  }
+
+  @Override
+  public void run() {
+    gcLog.info("Triggering gc on all repositories");
+    garbageCollectionFactory.create().run(
+        Lists.newArrayList(projectCache.all()));
+  }
+
+  @Override
+  public String toString() {
+    return "GC runner";
+  }
+}

gerrit-server/src/test/java/com/google/gerrit/server/config/ScheduleConfigTest.java

@@ -0,0 +1,83 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.config;
+
+import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.HOURS;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.junit.Assert.assertEquals;
+
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
+import org.joda.time.DateTime;
+import org.junit.Test;
+
+import java.text.MessageFormat;
+import java.util.concurrent.TimeUnit;
+
+public class ScheduleConfigTest {
+
+  // Friday June 13, 2014 10:00 UTC
+  private static final DateTime NOW = DateTime.parse("2014-06-13T10:00:00-00:00");
+
+  @Test
+  public void testInitialDelay() throws Exception {
+    assertEquals(ms(1, HOURS), initialDelay("11:00", "1h"));
+    assertEquals(ms(30, MINUTES), initialDelay("05:30", "1h"));
+    assertEquals(ms(30, MINUTES), initialDelay("09:30", "1h"));
+    assertEquals(ms(30, MINUTES), initialDelay("13:30", "1h"));
+    assertEquals(ms(59, MINUTES), initialDelay("13:59", "1h"));
+
+    assertEquals(ms(1, HOURS), initialDelay("11:00", "1d"));
+    assertEquals(ms(19, HOURS) + ms(30, MINUTES), initialDelay("05:30", "1d"));
+
+    assertEquals(ms(1, HOURS), initialDelay("11:00", "1w"));
+    assertEquals(ms(7, DAYS) - ms(4, HOURS) - ms(30, MINUTES),
+        initialDelay("05:30", "1w"));
+
+    assertEquals(ms(3, DAYS) + ms(1, HOURS), initialDelay("Mon 11:00", "1w"));
+    assertEquals(ms(1, HOURS), initialDelay("Fri 11:00", "1w"));
+
+    assertEquals(ms(1, HOURS), initialDelay("Mon 11:00", "1d"));
+    assertEquals(ms(23, HOURS), initialDelay("Mon 09:00", "1d"));
+    assertEquals(0, initialDelay("Mon 10:00", "1d"));
+  }
+
+  private static long initialDelay(String startTime, String interval)
+      throws ConfigInvalidException {
+    return config(startTime, interval).getInitialDelay();
+  }
+
+  private static ScheduleConfig config(String startTime, String interval)
+      throws ConfigInvalidException {
+    Config rc =
+        readConfig(MessageFormat.format(
+            "[section \"subsection\"]\nstartTime = {0}\ninterval = {1}\n",
+            startTime, interval));
+    return new ScheduleConfig(rc, "section", "subsection", NOW);
+  }
+
+  private static Config readConfig(String dat)
+      throws ConfigInvalidException {
+    Config config = new Config();
+    config.fromText(dat);
+    return config;
+  }
+
+  private static long ms(int cnt, TimeUnit unit) {
+    return MILLISECONDS.convert(cnt, unit);
+  }
+}

gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java

@@ -37,6 +37,7 @@ import com.google.gerrit.server.config.GerritServerConfigModule;
 import com.google.gerrit.server.config.MasterNodeStartup;
 import com.google.gerrit.server.config.SitePath;
 import com.google.gerrit.server.contact.HttpContactStoreConnection;
+import com.google.gerrit.server.git.GarbageCollectionRunner;
 import com.google.gerrit.server.git.LocalDiskRepositoryManager;
 import com.google.gerrit.server.git.ReceiveCommitsExecutorModule;
 import com.google.gerrit.server.git.WorkQueue;
@@ -310,6 +311,7 @@ public class WebAppInitializer extends GuiceServletContextListener
         bind(GerritUiOptions.class).toInstance(new GerritUiOptions(false));
       }
     });
+    modules.add(GarbageCollectionRunner.module());
     return cfgInjector.createChildInjector(modules);
   }