LFS protocol support

Expose LFS batch protocol [1] from the standard LFS URL:

  http://gerrit/path/to/project/info/lfs/*

as this is the default URL where the git-lfs client [2] expects the LFS
protocol. Therefore, no additional configuration of the git-lfs client
is required.

LFS protocol requests are forwarded to the configured lfs.plugin.
If no lfs.plugin is defined, Gerrit responds with "501 Not Implemented"
to all LFS protocol requests.

User authentication is the same as for git-over-http protocol. An
additional advantage of using the standard LFS URL is that the git-lfs
client will reuse the authentication credentials used for the
git-over-http because the schema and the host name are the same.

NOTE: Currently the standard git-lfs client only supports basic
authentication. This means that Gerrit must be configured to support
basic authentication.

[1] https://github.com/github/git-lfs/blob/master/docs/api/http-v1-batch.md
[2] https://git-lfs.github.com/

Change-Id: I1e3f29789d73af52c60d3788ee4fd2e7024b1c0c
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

Documentation/config-gerrit.txt

@@ -2851,6 +2851,17 @@ are specified in the link:#container[container section]:
   javaOptions = -Dcom.sun.jndi.ldap.connect.pool.timeout=300000
 ----
 
+[[lfs]]
+=== Section lfs
+
+[[lfs.plugin]]lfs.plugin::
++
+The name of a plugin which serves the LFS protocol on the
+`<project-name>/info/lfs/objects/batch` endpoint. When not configured Gerrit
+will respond with `501 Not Implemented` on LFS protocol requests.
++
+By default unset.
+
 [[log]]
 === Section log
 

Documentation/dev-plugins.txt

@@ -2046,6 +2046,60 @@ ProjectWebLinks will appear in the project list in the
 
 BranchWebLinks will appear in the branch list in the last column.
 
+[[lfs-extension]]
+== LFS Storage Plugins
+
+Gerrit provides an extension point that enables development of LFS (Large File
+Storage) storage plugins. Gerrit core exposes the default LFS protocol endpoint
+`<project-name>/info/lfs/objects/batch` and forwards the requests to the configured
+link:config-gerrit.html#lfs[lfs.plugin] plugin which implements the LFS protocol.
+By exposing the default LFS endpoint, the git-lfs client can be used without
+any configuration.
+
+[source, java]
+----
+/** Provide an LFS protocol implementation */
+import org.eclipse.jgit.lfs.server.LargeFileRepository;
+import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
+
+@Singleton
+public class LfsApiServlet extends LfsProtocolServlet {
+  private static final long serialVersionUID = 1L;
+
+  private final S3LargeFileRepository repository;
+
+  @Inject
+  LfsApiServlet(S3LargeFileRepository repository) {
+    this.repository = repository;
+  }
+
+  @Override
+  protected LargeFileRepository getLargeFileRepository() {
+    return repository;
+  }
+}
+
+/** Register the LfsApiServlet to listen on the default LFS protocol endpoint */
+import static com.google.gerrit.httpd.plugins.LfsPluginServlet.URL_REGEX;
+
+import com.google.gerrit.httpd.plugins.HttpPluginModule;
+
+public class HttpModule extends HttpPluginModule {
+
+  @Override
+  protected void configureServlets() {
+    serveRegex(URL_REGEX).with(LfsApiServlet.class);
+  }
+}
+
+/** Provide an implementation of the LargeFileRepository */
+import org.eclipse.jgit.lfs.server.s3.S3Repository;
+
+public class S3LargeFileRepository extends S3Repository {
+...
+}
+----
+
 [[documentation]]
 == Documentation
 

gerrit-httpd/src/main/java/com/google/gerrit/httpd/plugins/HttpPluginModule.java

@@ -32,6 +32,9 @@ public class HttpPluginModule extends ServletModule {
     bind(HttpPluginServlet.class);
     serveRegex("^/(?:a/)?plugins/(.*)?$").with(HttpPluginServlet.class);
 
+    bind(LfsPluginServlet.class);
+    serveRegex(LfsPluginServlet.URL_REGEX).with(LfsPluginServlet.class);
+
     bind(StartPluginListener.class)
       .annotatedWith(UniqueAnnotations.create())
       .to(HttpPluginServlet.class);
@@ -40,6 +43,14 @@ public class HttpPluginModule extends ServletModule {
       .annotatedWith(UniqueAnnotations.create())
       .to(HttpPluginServlet.class);
 
+    bind(StartPluginListener.class)
+      .annotatedWith(UniqueAnnotations.create())
+      .to(LfsPluginServlet.class);
+
+    bind(ReloadPluginListener.class)
+      .annotatedWith(UniqueAnnotations.create())
+      .to(LfsPluginServlet.class);
+
     bind(HttpModuleGenerator.class)
       .to(HttpAutoRegisterModuleGenerator.class);
 

gerrit-httpd/src/main/java/com/google/gerrit/httpd/plugins/LfsPluginServlet.java

@@ -0,0 +1,157 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.httpd.plugins;
+
+import static javax.servlet.http.HttpServletResponse.SC_NOT_IMPLEMENTED;
+
+import com.google.common.collect.Lists;
+import com.google.gerrit.extensions.registration.RegistrationHandle;
+import com.google.gerrit.httpd.resources.Resource;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.plugins.Plugin;
+import com.google.gerrit.server.plugins.ReloadPluginListener;
+import com.google.gerrit.server.plugins.StartPluginListener;
+import com.google.gwtexpui.server.CacheHeaders;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.google.inject.servlet.GuiceFilter;
+
+import org.eclipse.jgit.lib.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@Singleton
+public class LfsPluginServlet extends HttpServlet
+    implements StartPluginListener, ReloadPluginListener {
+  private static final long serialVersionUID = 1L;
+  private static final Logger log
+      = LoggerFactory.getLogger(LfsPluginServlet.class);
+
+  public static final String URL_REGEX =
+      "^(?:/a)?(?:/p/|/)(.+)(?:/info/lfs/objects/batch)$";
+
+  private List<Plugin> pending = Lists.newArrayList();
+  private final String pluginName;
+  private GuiceFilter filter;
+
+  @Inject
+  LfsPluginServlet(@GerritServerConfig Config cfg) {
+    this.pluginName = cfg.getString("lfs", null, "plugin");
+  }
+
+  @Override
+  protected void service(HttpServletRequest req, HttpServletResponse res)
+      throws ServletException, IOException {
+    if (filter == null) {
+      CacheHeaders.setNotCacheable(res);
+      res.sendError(SC_NOT_IMPLEMENTED);
+      return;
+    }
+
+    FilterChain chain = new FilterChain() {
+      @Override
+      public void doFilter(ServletRequest req, ServletResponse res)
+          throws IOException {
+        Resource.NOT_FOUND.send((HttpServletRequest) req, (HttpServletResponse) res);
+      }
+    };
+    if (filter != null) {
+      filter.doFilter(req, res, chain);
+    } else {
+      chain.doFilter(req, res);
+    }
+  }
+
+  @Override
+  public synchronized void init(ServletConfig config) throws ServletException {
+    super.init(config);
+
+    for (Plugin plugin : pending) {
+      install(plugin);
+    }
+    pending = null;
+  }
+
+  @Override
+  public synchronized void onStartPlugin(Plugin plugin) {
+    if (pending != null) {
+      pending.add(plugin);
+    } else {
+      install(plugin);
+    }
+  }
+
+  @Override
+  public void onReloadPlugin(Plugin oldPlugin, Plugin newPlugin) {
+    install(newPlugin);
+  }
+
+  private void install(Plugin plugin) {
+    if (!plugin.getName().equals(pluginName)) {
+      return;
+    }
+    filter = load(plugin);
+    plugin.add(new RegistrationHandle() {
+      @Override
+      public void remove() {
+        filter = null;
+      }
+    });
+  }
+
+  private GuiceFilter load(Plugin plugin) {
+    if (plugin.getHttpInjector() != null) {
+      final String name = plugin.getName();
+      final GuiceFilter filter;
+      try {
+        filter = plugin.getHttpInjector().getInstance(GuiceFilter.class);
+      } catch (RuntimeException e) {
+        log.warn(String.format("Plugin %s cannot load GuiceFilter", name), e);
+        return null;
+      }
+
+      try {
+        ServletContext ctx =
+            PluginServletContext.create(plugin, "/");
+        filter.init(new WrappedFilterConfig(ctx));
+      } catch (ServletException e) {
+        log.warn(String.format("Plugin %s failed to initialize HTTP", name), e);
+        return null;
+      }
+
+      plugin.add(new RegistrationHandle() {
+        @Override
+        public void remove() {
+          filter.destroy();
+        }
+      });
+      return filter;
+    }
+    return null;
+  }
+}

gerrit-httpd/src/test/java/com/google/gerrit/httpd/plugins/LfsPluginServletTest.java

@@ -0,0 +1,57 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.httpd.plugins;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import org.junit.Test;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class LfsPluginServletTest {
+
+  @Test
+  public void noLfsEndPoint_noMatch() {
+    Pattern p = Pattern.compile(LfsPluginServlet.URL_REGEX);
+    doesNotMatch(p, "/foo");
+    doesNotMatch(p, "/a/foo");
+    doesNotMatch(p, "/p/foo");
+    doesNotMatch(p, "/a/p/foo");
+
+    doesNotMatch(p, "/info/lfs/objects/batch");
+    doesNotMatch(p, "/info/lfs/objects/batch/foo");
+  }
+
+  @Test
+  public void matchingLfsEndpoint_projectNameCaptured() {
+    Pattern p = Pattern.compile(LfsPluginServlet.URL_REGEX);
+    matches(p, "/foo/bar/info/lfs/objects/batch", "foo/bar");
+    matches(p, "/a/foo/bar/info/lfs/objects/batch", "foo/bar");
+    matches(p, "/p/foo/bar/info/lfs/objects/batch", "foo/bar");
+    matches(p, "/a/p/foo/bar/info/lfs/objects/batch", "foo/bar");
+  }
+
+  private void doesNotMatch(Pattern p, String input) {
+    Matcher m = p.matcher(input);
+    assertThat(m.matches()).isFalse();
+  }
+
+  private void matches(Pattern p, String input, String expectedProjectName) {
+    Matcher m = p.matcher(input);
+    assertThat(m.matches()).isTrue();
+    assertThat(m.group(1)).isEqualTo(expectedProjectName);
+  }
+}