Merge "Allow to schedule a task to run gc for all projects"

Documentation/config-gerrit.txt

@@ -1343,6 +1343,68 @@ screen:
 If `download.archive` is not specified defaults to all archive
 commands. Set to `off` or empty string to disable.
 
+[[gc]]
+=== Section gc
+
+This section allows to configure the git garbage collection and schedules it
+to run periodically. It will be triggered and executed sequentially for all
+projects.
+
+[[gc.startTime]]gc.startTime::
++
+Start time to define the first execution of the git garbage collection.
+If the configured `'gc.interval'` is shorter than `'gc.startTime - now'`
+the start time will be preponed by the maximum integral multiple of
+`'gc.interval'` so that the start time is still in the future.
++
+----
+<day of week> <hours>:<minutes>
+or
+<hours>:<minutes>
+
+<day of week> : Mon, Tue, Wed, Thu, Fri, Sat, Sun
+<hours>       : 00-23
+<minutes>     : 0-59
+----
+
+
+[[gc.interval]]gc.interval::
++
+Interval for periodic repetition of triggering the git garbage collection.
+The interval must be larger than zero. The following suffixes are supported
+to define the time unit for the interval:
++
+* `s, sec, second, seconds`
+* `m, min, minute, minutes`
+* `h, hr, hour, hours`
+* `d, day, days`
+* `w, week, weeks` (`1 week` is treated as `7 days`)
+* `mon, month, months` (`1 month` is treated as `30 days`)
+* `y, year, years` (`1 year` is treated as `365 days`)
+
+Examples::
++
+----
+gc.startTime = Fri 10:30
+gc.interval  = 2 day
+----
++
+Assuming the server is started on Mon 7:00 -> `'startTime - now = 4 days 3:30 hours'`.
+This is larger than the interval hence prepone the start time
+by the maximum integral multiple of the interval so that start
+time is still in the future, i.e. prepone by 4 days. This yields
+a start time of Mon 10:30, next executions are Wed 10:30, Fri 10:30
+etc.
++
+----
+gc.startTime = 6:00
+gc.interval = 1 day
+----
++
+Assuming the server is started on Mon 7:00 this yields the first run on next Tuesday
+at 6:00 and a repetition interval of 1 day.
+
+
 [[gerrit]]
 === Section gerrit
 

gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java

@@ -53,6 +53,7 @@ import com.google.gerrit.server.config.GerritGlobalModule;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.MasterNodeStartup;
 import com.google.gerrit.server.contact.HttpContactStoreConnection;
+import com.google.gerrit.server.git.GarbageCollectionRunner;
 import com.google.gerrit.server.git.ReceiveCommitsExecutorModule;
 import com.google.gerrit.server.git.WorkQueue;
 import com.google.gerrit.server.index.IndexModule;
@@ -353,6 +354,7 @@ public class Daemon extends SiteProgram {
         bind(GerritUiOptions.class).toInstance(new GerritUiOptions(headless));
       }
     });
+    modules.add(GarbageCollectionRunner.module());
     return cfgInjector.createChildInjector(modules);
   }
 

gerrit-server/src/main/java/com/google/gerrit/server/config/GcConfig.java

@@ -0,0 +1,36 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.config;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.ConfigConstants;
+
+@Singleton
+public class GcConfig {
+  private final ScheduleConfig scheduleConfig;
+
+  @Inject
+  GcConfig(@GerritServerConfig Config cfg) {
+    scheduleConfig = new ScheduleConfig(cfg, ConfigConstants.CONFIG_GC_SECTION);
+  }
+
+  public ScheduleConfig getScheduleConfig() {
+    return scheduleConfig;
+  }
+
+}

gerrit-server/src/main/java/com/google/gerrit/server/config/GerritGlobalModule.java

@@ -218,6 +218,8 @@ public class GerritGlobalModule extends FactoryModule {
     bind(EventFactory.class);
     bind(TransferConfig.class);
 
+    bind(GcConfig.class);
+
     bind(ApprovalsUtil.class);
     bind(ChangeMergeQueue.class).in(SINGLETON);
     bind(MergeQueue.class).to(ChangeMergeQueue.class).in(SINGLETON);

gerrit-server/src/main/java/com/google/gerrit/server/config/ScheduleConfig.java

@@ -0,0 +1,128 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.config;
+
+import org.eclipse.jgit.lib.Config;
+import org.joda.time.DateTime;
+import org.joda.time.LocalDateTime;
+import org.joda.time.LocalTime;
+import org.joda.time.MutableDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.ISODateTimeFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.MessageFormat;
+import java.util.concurrent.TimeUnit;
+
+public class ScheduleConfig {
+  private static final Logger log = LoggerFactory
+      .getLogger(ScheduleConfig.class);
+  public static final long MISSING_CONFIG = -1L;
+  public static final long INVALID_CONFIG = -2L;
+  private static final String KEY_INTERVAL = "interval";
+  private static final String KEY_STARTTIME = "startTime";
+
+  private final long initialDelay;
+  private final long interval;
+
+  public ScheduleConfig(Config rc, String section) {
+    this(rc, section, null);
+  }
+
+  public ScheduleConfig(Config rc, String section, String subsection) {
+    this(rc, section, subsection, DateTime.now());
+  }
+
+  /* For testing we need to be able to pass now */
+  ScheduleConfig(Config rc, String section, String subsection, DateTime now) {
+    this.interval = interval(rc, section, subsection);
+    this.initialDelay = initialDelay(rc, section, subsection, now, interval);
+  }
+
+  public long getInitialDelay() {
+    return initialDelay;
+  }
+
+  public long getInterval() {
+    return interval;
+  }
+
+  private static long interval(Config rc, String section, String subsection) {
+    long interval = MISSING_CONFIG;
+    try {
+      interval =
+          ConfigUtil.getTimeUnit(rc, section, subsection, KEY_INTERVAL, -1,
+              TimeUnit.MILLISECONDS);
+      if (interval == MISSING_CONFIG) {
+        log.info(MessageFormat.format(
+            "{0} schedule parameter \"{0}.{1}\" is not configured", section,
+            KEY_INTERVAL));
+      }
+    } catch (IllegalArgumentException e) {
+      log.error(MessageFormat.format(
+          "Invalid {0} schedule parameter \"{0}.{1}\"", section, KEY_INTERVAL),
+          e);
+      interval = INVALID_CONFIG;
+    }
+    return interval;
+  }
+
+  private static long initialDelay(Config rc, String section,
+      String subsection, DateTime now, long interval) {
+    long delay = MISSING_CONFIG;
+    String start = rc.getString(section, subsection, KEY_STARTTIME);
+    try {
+      if (start != null) {
+        DateTimeFormatter formatter = ISODateTimeFormat.hourMinute();
+        MutableDateTime startTime = now.toMutableDateTime();
+        LocalTime firstStartTime = null;
+        LocalDateTime firstStartDateTime = null;
+        try {
+          firstStartTime = formatter.parseLocalTime(start);
+          startTime.hourOfDay().set(firstStartTime.getHourOfDay());
+          startTime.minuteOfHour().set(firstStartTime.getMinuteOfHour());
+        } catch (IllegalArgumentException e1) {
+          formatter = DateTimeFormat.forPattern("E HH:mm");
+          firstStartDateTime = formatter.parseLocalDateTime(start);
+          startTime.dayOfWeek().set(firstStartDateTime.getDayOfWeek());
+          startTime.hourOfDay().set(firstStartDateTime.getHourOfDay());
+          startTime.minuteOfHour().set(firstStartDateTime.getMinuteOfHour());
+        }
+        startTime.secondOfMinute().set(0);
+        startTime.millisOfSecond().set(0);
+        while (startTime.isBefore(now)) {
+          startTime.add(interval);
+        }
+        while (startTime.getMillis() - now.getMillis() > interval) {
+          startTime.add(-interval);
+        }
+        delay = startTime.getMillis() - now.getMillis();
+      } else {
+        log.info(MessageFormat.format(
+            "{0} schedule parameter \"{0}.{1}\" is not configured", section,
+            KEY_STARTTIME));
+      }
+    } catch (IllegalArgumentException e2) {
+      log.error(
+          MessageFormat.format("Invalid {0} schedule parameter \"{0}.{1}\"",
+              section, KEY_STARTTIME), e2);
+      delay = INVALID_CONFIG;
+    }
+    return delay;
+  }
+
+}

gerrit-server/src/main/java/com/google/gerrit/server/git/GarbageCollectionRunner.java

@@ -0,0 +1,103 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.git;
+
+import static com.google.gerrit.server.config.ScheduleConfig.MISSING_CONFIG;
+
+import com.google.common.collect.Lists;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.server.config.GcConfig;
+import com.google.gerrit.server.config.ScheduleConfig;
+import com.google.gerrit.server.project.ProjectCache;
+import com.google.inject.Inject;
+import com.google.inject.Module;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/** Runnable to enable scheduling gc to run periodically */
+public class GarbageCollectionRunner implements Runnable {
+  public static final String LOG_NAME = "gc_log";
+  private static final Logger gcLog = LoggerFactory.getLogger(LOG_NAME);
+
+  public static Module module() {
+    return new LifecycleModule() {
+
+      @Override
+      protected void configure() {
+        listener().to(Lifecycle.class);
+      }
+    };
+  }
+
+  static class Lifecycle implements LifecycleListener {
+    private final WorkQueue queue;
+    private final GarbageCollectionRunner gcRunner;
+    private final GcConfig gcConfig;
+
+    @Inject
+    Lifecycle(WorkQueue queue, GarbageCollectionRunner gcRunner,
+        GcConfig config) {
+      this.queue = queue;
+      this.gcRunner = gcRunner;
+      this.gcConfig = config;
+    }
+
+    @Override
+    public void start() {
+      ScheduleConfig scheduleConfig = gcConfig.getScheduleConfig();
+      long interval = scheduleConfig.getInterval();
+      long delay = scheduleConfig.getInitialDelay();
+      if (delay == MISSING_CONFIG && interval == MISSING_CONFIG) {
+        gcLog.info("Ignoring missing gc schedule configuration");
+      } else if (delay < 0 || interval <= 0) {
+        gcLog.warn("Ignoring invalid gc schedule configuration");
+      } else {
+        queue.getDefaultQueue().scheduleWithFixedDelay(gcRunner, delay,
+            interval, TimeUnit.MILLISECONDS);
+      }
+    }
+
+    @Override
+    public void stop() {
+      // handled by WorkQueue.stop() already
+    }
+  }
+
+  private final GarbageCollection.Factory garbageCollectionFactory;
+  private final ProjectCache projectCache;
+
+  @Inject
+  GarbageCollectionRunner(GarbageCollection.Factory garbageCollectionFactory,
+      ProjectCache projectCache) {
+    this.garbageCollectionFactory = garbageCollectionFactory;
+    this.projectCache = projectCache;
+  }
+
+  @Override
+  public void run() {
+    gcLog.info("Triggering gc on all repositories");
+    garbageCollectionFactory.create().run(
+        Lists.newArrayList(projectCache.all()));
+  }
+
+  @Override
+  public String toString() {
+    return "GC runner";
+  }
+}

gerrit-server/src/test/java/com/google/gerrit/server/config/ScheduleConfigTest.java

@@ -0,0 +1,83 @@
+// Copyright (C) 2014 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.config;
+
+import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.HOURS;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.junit.Assert.assertEquals;
+
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lib.Config;
+import org.joda.time.DateTime;
+import org.junit.Test;
+
+import java.text.MessageFormat;
+import java.util.concurrent.TimeUnit;
+
+public class ScheduleConfigTest {
+
+  // Friday June 13, 2014 10:00 UTC
+  private static final DateTime NOW = DateTime.parse("2014-06-13T10:00:00-00:00");
+
+  @Test
+  public void testInitialDelay() throws Exception {
+    assertEquals(ms(1, HOURS), initialDelay("11:00", "1h"));
+    assertEquals(ms(30, MINUTES), initialDelay("05:30", "1h"));
+    assertEquals(ms(30, MINUTES), initialDelay("09:30", "1h"));
+    assertEquals(ms(30, MINUTES), initialDelay("13:30", "1h"));
+    assertEquals(ms(59, MINUTES), initialDelay("13:59", "1h"));
+
+    assertEquals(ms(1, HOURS), initialDelay("11:00", "1d"));
+    assertEquals(ms(19, HOURS) + ms(30, MINUTES), initialDelay("05:30", "1d"));
+
+    assertEquals(ms(1, HOURS), initialDelay("11:00", "1w"));
+    assertEquals(ms(7, DAYS) - ms(4, HOURS) - ms(30, MINUTES),
+        initialDelay("05:30", "1w"));
+
+    assertEquals(ms(3, DAYS) + ms(1, HOURS), initialDelay("Mon 11:00", "1w"));
+    assertEquals(ms(1, HOURS), initialDelay("Fri 11:00", "1w"));
+
+    assertEquals(ms(1, HOURS), initialDelay("Mon 11:00", "1d"));
+    assertEquals(ms(23, HOURS), initialDelay("Mon 09:00", "1d"));
+    assertEquals(0, initialDelay("Mon 10:00", "1d"));
+  }
+
+  private static long initialDelay(String startTime, String interval)
+      throws ConfigInvalidException {
+    return config(startTime, interval).getInitialDelay();
+  }
+
+  private static ScheduleConfig config(String startTime, String interval)
+      throws ConfigInvalidException {
+    Config rc =
+        readConfig(MessageFormat.format(
+            "[section \"subsection\"]\nstartTime = {0}\ninterval = {1}\n",
+            startTime, interval));
+    return new ScheduleConfig(rc, "section", "subsection", NOW);
+  }
+
+  private static Config readConfig(String dat)
+      throws ConfigInvalidException {
+    Config config = new Config();
+    config.fromText(dat);
+    return config;
+  }
+
+  private static long ms(int cnt, TimeUnit unit) {
+    return MILLISECONDS.convert(cnt, unit);
+  }
+}

gerrit-war/src/main/java/com/google/gerrit/httpd/WebAppInitializer.java

@@ -37,6 +37,7 @@ import com.google.gerrit.server.config.GerritServerConfigModule;
 import com.google.gerrit.server.config.MasterNodeStartup;
 import com.google.gerrit.server.config.SitePath;
 import com.google.gerrit.server.contact.HttpContactStoreConnection;
+import com.google.gerrit.server.git.GarbageCollectionRunner;
 import com.google.gerrit.server.git.LocalDiskRepositoryManager;
 import com.google.gerrit.server.git.ReceiveCommitsExecutorModule;
 import com.google.gerrit.server.git.WorkQueue;
@@ -310,6 +311,7 @@ public class WebAppInitializer extends GuiceServletContextListener
         bind(GerritUiOptions.class).toInstance(new GerritUiOptions(false));
       }
     });
+    modules.add(GarbageCollectionRunner.module());
     return cfgInjector.createChildInjector(modules);
   }