Serve bower_components from buck-out with --polygerrit-dev

Rather than depending on the bower_components directory built by
`bower install`, use the Buck build and serve components directly from
polygerrit_components.bower_components.zip. Add a filter in front of
PolyGerrit index paths to recompile bower_components.

The end result is developers can run:
  buck build polygerrit && \
  java -jar buck-out/.../polygerrit.war --polygerrit-dev ...
to start up a PolyGerrit dev server serving local content, without the
need for Eclipse, and that doesn't need to be restarted even if
bower_component deps change.

Change-Id: I8658a2b03ff8ecb6824092e02411ba3b67d37569

gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/BowerComponentsServlet.java

@@ -0,0 +1,48 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.httpd.raw;
+
+import com.google.common.cache.Cache;
+import com.google.gerrit.launcher.GerritLauncher;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+class BowerComponentsServlet extends ResourceServlet {
+  private static final long serialVersionUID = 1L;
+
+  static Path getZipPath(Path buckOut) {
+    if (buckOut == null) {
+      return null;
+    }
+    return buckOut.resolve("gen")
+        .resolve("polygerrit-ui")
+        .resolve("polygerrit_components")
+        .resolve("polygerrit_components.bower_components.zip");
+  }
+
+  private final Path zip;
+
+  BowerComponentsServlet(Cache<Path, Resource> cache, Path buckOut) {
+    super(cache, true);
+    this.zip = getZipPath(buckOut);
+  }
+
+  @Override
+  protected Path getResourcePath(String pathInfo) throws IOException {
+    return GerritLauncher.getZipFileSystem(zip)
+        .getPath("bower_components/" + pathInfo);
+  }
+}

gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/BuckUtils.java

@@ -0,0 +1,117 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.httpd.raw;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.escape.Escaper;
+import com.google.common.html.HtmlEscapers;
+import com.google.common.io.ByteStreams;
+import com.google.gerrit.common.TimeUtil;
+import com.google.gwtexpui.server.CacheHeaders;
+
+import org.eclipse.jgit.util.RawParseUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.io.PrintWriter;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+
+import javax.servlet.http.HttpServletResponse;
+
+class BuckUtils {
+  private static final Logger log =
+      LoggerFactory.getLogger(BuckUtils.class);
+
+  static void build(Path root, Path gen, String target)
+      throws IOException, BuildFailureException {
+    log.info("buck build " + target);
+    Properties properties = loadBuckProperties(gen);
+    String buck = MoreObjects.firstNonNull(properties.getProperty("buck"), "buck");
+    ProcessBuilder proc = new ProcessBuilder(buck, "build", target)
+        .directory(root.toFile())
+        .redirectErrorStream(true);
+    if (properties.containsKey("PATH")) {
+      proc.environment().put("PATH", properties.getProperty("PATH"));
+    }
+    long start = TimeUtil.nowMs();
+    Process rebuild = proc.start();
+    byte[] out;
+    try (InputStream in = rebuild.getInputStream()) {
+      out = ByteStreams.toByteArray(in);
+    } finally {
+      rebuild.getOutputStream().close();
+    }
+
+    int status;
+    try {
+      status = rebuild.waitFor();
+    } catch (InterruptedException e) {
+      throw new InterruptedIOException("interrupted waiting for " + buck);
+    }
+    if (status != 0) {
+      throw new BuildFailureException(out);
+    }
+
+    long time = TimeUtil.nowMs() - start;
+    log.info(String.format("UPDATED    %s in %.3fs", target, time / 1000.0));
+  }
+
+  private static Properties loadBuckProperties(Path gen)
+      throws FileNotFoundException, IOException {
+    Properties properties = new Properties();
+    try (InputStream in = new FileInputStream(
+        gen.resolve(Paths.get("tools/buck/buck.properties")).toFile())) {
+      properties.load(in);
+    }
+    return properties;
+  }
+
+  static void displayFailure(String rule, byte[] why, HttpServletResponse res)
+      throws IOException {
+    res.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+    res.setContentType("text/html");
+    res.setCharacterEncoding(UTF_8.name());
+    CacheHeaders.setNotCacheable(res);
+
+    Escaper html = HtmlEscapers.htmlEscaper();
+    try (PrintWriter w = res.getWriter()) {
+      w.write("<html><title>BUILD FAILED</title><body>");
+      w.format("<h1>%s FAILED</h1>", html.escape(rule));
+      w.write("<pre>");
+      w.write(html.escape(RawParseUtils.decode(why)));
+      w.write("</pre>");
+      w.write("</body></html>");
+    }
+  }
+
+  static class BuildFailureException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    final byte[] why;
+
+    BuildFailureException(byte[] why) {
+      this.why = why;
+    }
+  }
+}

gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/RebuildBowerComponentsFilter.java

@@ -0,0 +1,74 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.httpd.raw;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.gerrit.httpd.raw.BuckUtils.BuildFailureException;
+import com.google.gerrit.launcher.GerritLauncher;
+import com.google.inject.Singleton;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+
+@Singleton
+class RebuildBowerComponentsFilter implements Filter {
+  private static final String TARGET = "//polygerrit-ui:polygerrit_components";
+
+  private final Path gen;
+  private final Path root;
+  private final Path zip;
+
+  RebuildBowerComponentsFilter(Path buckOut) {
+    gen = buckOut.resolve("gen");
+    root = buckOut.getParent();
+    zip = BowerComponentsServlet.getZipPath(buckOut);
+  }
+
+  @Override
+  public synchronized void doFilter(ServletRequest sreq, ServletResponse sres,
+      FilterChain chain) throws IOException, ServletException {
+    HttpServletResponse res = (HttpServletResponse) sres;
+    try {
+      BuckUtils.build(root, gen, TARGET);
+    } catch (BuildFailureException e) {
+      BuckUtils.displayFailure(TARGET, e.why, res);
+      return;
+    }
+    if (!Files.exists(zip)) {
+      String msg = "`buck build` did not produce " + zip.toAbsolutePath();
+      BuckUtils.displayFailure(TARGET, msg.getBytes(UTF_8), res);
+    }
+    GerritLauncher.reloadZipFileSystem(zip);
+    chain.doFilter(sreq, sres);
+  }
+
+  @Override
+  public void init(FilterConfig config) throws ServletException {
+  }
+
+  @Override
+  public void destroy() {
+  }
+}

gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/RecompileGwtUiFilter.java

@@ -14,33 +14,16 @@
 
 package com.google.gerrit.httpd.raw;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
+import com.google.gerrit.httpd.raw.BuckUtils.BuildFailureException;
-
-import com.google.common.base.MoreObjects;
-import com.google.common.escape.Escaper;
-import com.google.common.html.HtmlEscapers;
-import com.google.common.io.ByteStreams;
-import com.google.gerrit.common.TimeUtil;
 import com.google.gwtexpui.linker.server.UserAgentRule;
-import com.google.gwtexpui.server.CacheHeaders;
-
-import org.eclipse.jgit.util.RawParseUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InterruptedIOException;
-import java.io.PrintWriter;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Enumeration;
 import java.util.HashSet;
-import java.util.Properties;
 import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
@@ -55,9 +38,6 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 class RecompileGwtUiFilter implements Filter {
-  private static final Logger log =
-      LoggerFactory.getLogger(RecompileGwtUiFilter.class);
-
   private final boolean gwtuiRecompile =
       System.getProperty("gerrit.disable-gwtui-recompile") == null;
   private final UserAgentRule rule = new UserAgentRule();
@@ -92,9 +72,9 @@ class RecompileGwtUiFilter implements Filter {
 
       synchronized (this) {
         try {
-          build(root, gen, rule);
+          BuckUtils.build(root, gen, rule);
         } catch (BuildFailureException e) {
-          displayFailure(rule, e.why, (HttpServletResponse) res);
+          BuckUtils.displayFailure(rule, e.why, (HttpServletResponse) res);
           return;
         }
 
@@ -109,24 +89,6 @@ class RecompileGwtUiFilter implements Filter {
     chain.doFilter(request, res);
   }
 
-  private void displayFailure(String rule, byte[] why, HttpServletResponse res)
-      throws IOException {
-    res.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
-    res.setContentType("text/html");
-    res.setCharacterEncoding(UTF_8.name());
-    CacheHeaders.setNotCacheable(res);
-
-    Escaper html = HtmlEscapers.htmlEscaper();
-    try (PrintWriter w = res.getWriter()) {
-      w.write("<html><title>BUILD FAILED</title><body>");
-      w.format("<h1>%s FAILED</h1>", html.escape(rule));
-      w.write("<pre>");
-      w.write(html.escape(RawParseUtils.decode(why)));
-      w.write("</pre>");
-      w.write("</body></html>");
-    }
-  }
-
   @Override
   public void init(FilterConfig config) {
   }
@@ -166,59 +128,6 @@ class RecompileGwtUiFilter implements Filter {
     }
   }
 
-  private static void build(Path root, Path gen, String target)
-      throws IOException, BuildFailureException {
-    log.info("buck build " + target);
-    Properties properties = loadBuckProperties(gen);
-    String buck = MoreObjects.firstNonNull(properties.getProperty("buck"), "buck");
-    ProcessBuilder proc = new ProcessBuilder(buck, "build", target)
-        .directory(root.toFile())
-        .redirectErrorStream(true);
-    if (properties.containsKey("PATH")) {
-      proc.environment().put("PATH", properties.getProperty("PATH"));
-    }
-    long start = TimeUtil.nowMs();
-    Process rebuild = proc.start();
-    byte[] out;
-    try (InputStream in = rebuild.getInputStream()) {
-      out = ByteStreams.toByteArray(in);
-    } finally {
-      rebuild.getOutputStream().close();
-    }
-
-    int status;
-    try {
-      status = rebuild.waitFor();
-    } catch (InterruptedException e) {
-      throw new InterruptedIOException("interrupted waiting for " + buck);
-    }
-    if (status != 0) {
-      throw new BuildFailureException(out);
-    }
-
-    long time = TimeUtil.nowMs() - start;
-    log.info(String.format("UPDATED    %s in %.3fs", target, time / 1000.0));
-  }
-
-  private static Properties loadBuckProperties(Path gen)
-      throws FileNotFoundException, IOException {
-    Properties properties = new Properties();
-    try (InputStream in = new FileInputStream(
-        gen.resolve(Paths.get("tools/buck/buck.properties")).toFile())) {
-      properties.load(in);
-    }
-    return properties;
-  }
-
-  @SuppressWarnings("serial")
-  private static class BuildFailureException extends Exception {
-    final byte[] why;
-
-    BuildFailureException(byte[] why) {
-      this.why = why;
-    }
-  }
-
   private static void mkdir(File dir) throws IOException {
     if (!dir.isDirectory()) {
       mkdir(dir.getParentFile());

gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java

@@ -108,8 +108,9 @@ public abstract class ResourceServlet extends HttpServlet {
    *
    * @param pathInfo result of {@link HttpServletRequest#getPathInfo()}.
    * @return path where static content can be found.
+   * @throws IOException if an error occurred resolving the resource.
    */
-  protected abstract Path getResourcePath(String pathInfo);
+  protected abstract Path getResourcePath(String pathInfo) throws IOException;
 
   protected FileTime getLastModifiedTime(Path p) throws IOException {
     return Files.getLastModifiedTime(p);
@@ -198,7 +199,7 @@ public abstract class ResourceServlet extends HttpServlet {
     try {
       Path p = getResourcePath(name);
       return cache.get(p, newLoader(p));
-    } catch (ExecutionException e) {
+    } catch (ExecutionException | IOException e) {
       log.warn(String.format("Cannot load static resource %s", name), e);
       return null;
     }

gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java

@@ -17,6 +17,7 @@ package com.google.gerrit.httpd.raw;
 import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.common.cache.Cache;
+import com.google.common.collect.ImmutableList;
 import com.google.gerrit.httpd.GerritOptions;
 import com.google.gerrit.httpd.raw.ResourceServlet.Resource;
 import com.google.gerrit.launcher.GerritLauncher;
@@ -39,10 +40,24 @@ import java.nio.file.Path;
 import javax.servlet.http.HttpServlet;
 
 public class StaticModule extends ServletModule {
-  private static final String GWT_UI_SERVLET = "GwtUiServlet";
-  private static final String BOWER_SERVLET = "BowerServlet";
   public static final String CACHE = "static_content";
 
+  public static final ImmutableList<String> POLYGERRIT_INDEX_PATHS =
+      ImmutableList.of(
+        "/",
+        "/c/*",
+        "/q/*",
+        "/x/*",
+        "/admin/*",
+        "/dashboard/*",
+        "/settings/*",
+        // TODO(dborowitz): These fragments conflict with the REST API
+        // namespace, so they will need to use a different path.
+        "/groups/*",
+        "/projects/*");
+
+  private static final String GWT_UI_SERVLET = "GwtUiServlet";
+
   private final GerritOptions options;
   private Paths paths;
 
@@ -104,21 +119,26 @@ public class StaticModule extends ServletModule {
   private class PolyGerritUiModule extends ServletModule {
     @Override
     public void configureServlets() {
-      serve("/").with(PolyGerritUiIndexServlet.class);
+      Path buckOut = getPaths().buckOut;
-      serve("/c/*").with(PolyGerritUiIndexServlet.class);
+      if (buckOut != null) {
-      serve("/q/*").with(PolyGerritUiIndexServlet.class);
+        RebuildBowerComponentsFilter rebuildFilter =
-      serve("/x/*").with(PolyGerritUiIndexServlet.class);
+            new RebuildBowerComponentsFilter(buckOut);
-      serve("/admin/*").with(PolyGerritUiIndexServlet.class);
+        for (String p : POLYGERRIT_INDEX_PATHS) {
-      serve("/dashboard/*").with(PolyGerritUiIndexServlet.class);
+          // Rebuilding bower_components once per load on the index request,
-      serve("/settings/*").with(PolyGerritUiIndexServlet.class);
+          // is sufficient, since it will finish building before attempting to
-      // TODO(dborowitz): These fragments conflict with the REST API namespace,
+          // access any bower_components resources. Plus it saves contention and
-      // so they will need to use a different path.
+          // extraneous buck builds.
-      //serve("/groups/*").with(PolyGerritUiIndexServlet.class);
+          filter(p).through(rebuildFilter);
-      //serve("/projects/*").with(PolyGerritUiIndexServlet.class);
+        }
+        serve("/bower_components/*").with(BowerComponentsServlet.class);
+      } else {
+        // In the war case, bower_components are either inlined by vulcanize, or
+        // live under /polygerrit_ui in the war file, so we don't need a
+        // separate servlet.
+      }
 
-      if (getPaths().warFs == null) {
+      for (String p : POLYGERRIT_INDEX_PATHS) {
-        serve("/bower_components/*").with(
+        serve(p).with(PolyGerritUiIndexServlet.class);
-            Key.get(PolyGerritUiServlet.class, Names.named(BOWER_SERVLET)));
       }
       serve("/*").with(PolyGerritUiServlet.class);
     }
@@ -139,11 +159,9 @@ public class StaticModule extends ServletModule {
 
     @Provides
     @Singleton
-    @Named(BOWER_SERVLET)
+    BowerComponentsServlet getBowerComponentsServlet(
-    PolyGerritUiServlet getPolyGerritUiBowerServlet(
         @Named(CACHE) Cache<Path, Resource> cache) {
-      return new PolyGerritUiServlet(cache,
+      return new BowerComponentsServlet(cache, getPaths().buckOut);
-          polyGerritBasePath().resolveSibling("bower_components"));
     }
 
     private Path polyGerritBasePath() {
@@ -159,7 +177,7 @@ public class StaticModule extends ServletModule {
     }
   }
 
-  private static class Paths {
+  private class Paths {
     private final FileSystem warFs;
     private final Path buckOut;
     private final Path unpackedWar;
@@ -170,6 +188,9 @@ public class StaticModule extends ServletModule {
         if (warFs == null) {
           buckOut = getDeveloperBuckOut();
           unpackedWar = makeWarTempDir();
+        } else if (options.forcePolyGerritDev()) {
+          buckOut = getDeveloperBuckOut();
+          unpackedWar = null;
         } else {
           buckOut = null;
           unpackedWar = null;
@@ -180,7 +201,7 @@ public class StaticModule extends ServletModule {
       }
     }
 
-    private static FileSystem getDistributionArchive() throws IOException {
+    private FileSystem getDistributionArchive() throws IOException {
       File war;
       try {
         war = GerritLauncher.getDistributionArchive();
@@ -198,7 +219,7 @@ public class StaticModule extends ServletModule {
       return GerritLauncher.getZipFileSystem(war.toPath());
     }
 
-    private static Path getDeveloperBuckOut() {
+    private Path getDeveloperBuckOut() {
       try {
         return GerritLauncher.getDeveloperBuckOut();
       } catch (FileNotFoundException e) {
@@ -206,7 +227,7 @@ public class StaticModule extends ServletModule {
       }
     }
 
-    private static Path makeWarTempDir() {
+    private Path makeWarTempDir() {
       // Obtain our local temporary directory, but it comes back as a file
       // so we have to switch it to be a directory post creation.
       //

gerrit-launcher/src/main/java/com/google/gerrit/launcher/GerritLauncher.java

@@ -334,14 +334,44 @@ public final class GerritLauncher {
     zip = zip.toRealPath();
     FileSystem zipFs = zipFileSystems.get(zip);
     if (zipFs == null) {
-      zipFs = FileSystems.newFileSystem(
+      zipFs = newZipFileSystem(zip);
-          URI.create("jar:" + zip.toUri()),
-          Collections.<String, String> emptyMap());
       zipFileSystems.put(zip, zipFs);
     }
     return zipFs;
   }
 
+  /**
+   * Reload the zip {@link FileSystem} for a path.
+   * <p>
+   * <strong>Warning</strong>: This calls {@link FileSystem#close()} on any
+   * previously open instance of the filesystem at this path, which may cause
+   * {@code IOException}s in any open path handles created with the old
+   * filesystem. Use with caution.
+   *
+   * @param zip path to zip file.
+   * @return reloaded filesystem instance.
+   * @throws IOException if there was an error reading the zip file.
+   */
+  public static synchronized FileSystem reloadZipFileSystem(Path zip)
+      throws IOException {
+    // FileSystems canonicalizes the path, so we should too.
+    zip = zip.toRealPath();
+    @SuppressWarnings("resource") // Caching resource for later use.
+    FileSystem zipFs = zipFileSystems.get(zip);
+    if (zipFs != null) {
+      zipFs.close();
+    }
+    zipFs = newZipFileSystem(zip);
+    zipFileSystems.put(zip, zipFs);
+    return zipFs;
+  }
+
+  private static FileSystem newZipFileSystem(Path zip) throws IOException {
+    return FileSystems.newFileSystem(
+        URI.create("jar:" + zip.toUri()),
+        Collections.<String, String> emptyMap());
+  }
+
   private static File locateMyArchive() throws FileNotFoundException {
     final ClassLoader myCL = GerritLauncher.class.getClassLoader();
     final String myName =