opendev/gerrit
Code Issues Proposed changes

Serve bower_components from buck-out with --polygerrit-dev

Browse Source 
Rather than depending on the bower_components directory built by
`bower install`, use the Buck build and serve components directly from
polygerrit_components.bower_components.zip. Add a filter in front of
PolyGerrit index paths to recompile bower_components.

The end result is developers can run:
  buck build polygerrit && \
  java -jar buck-out/.../polygerrit.war --polygerrit-dev ...
to start up a PolyGerrit dev server serving local content, without the
need for Eclipse, and that doesn't need to be restarted even if
bower_component deps change.

Change-Id: I8658a2b03ff8ecb6824092e02411ba3b67d37569
This commit is contained in:
Dave Borowitz 2015-11-16 09:44:44 -05:00
parent 6a86b1bebe
commit 788cb87f97
7 changed files with 323 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/BowerComponentsServlet.java Normal file
View File

@ -0,0 +1,48 @@
// Copyright (C) 2015 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.httpd.raw;
import com.google.common.cache.Cache;
import com.google.gerrit.launcher.GerritLauncher;
import java.io.IOException;
import java.nio.file.Path;
class BowerComponentsServlet extends ResourceServlet {
private static final long serialVersionUID = 1L;
static Path getZipPath(Path buckOut) {
if (buckOut == null) {
return null;
}
return buckOut.resolve("gen")
.resolve("polygerrit-ui")
.resolve("polygerrit_components")
.resolve("polygerrit_components.bower_components.zip");
}
private final Path zip;
BowerComponentsServlet(Cache<Path, Resource> cache, Path buckOut) {
super(cache, true);
this.zip = getZipPath(buckOut);
}
@Override
protected Path getResourcePath(String pathInfo) throws IOException {
return GerritLauncher.getZipFileSystem(zip)
.getPath("bower_components/" + pathInfo);
}
}

117
gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/BuckUtils.java Normal file
View File

@ -0,0 +1,117 @@
// Copyright (C) 2015 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.httpd.raw;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.base.MoreObjects;
import com.google.common.escape.Escaper;
import com.google.common.html.HtmlEscapers;
import com.google.common.io.ByteStreams;
import com.google.gerrit.common.TimeUtil;
import com.google.gwtexpui.server.CacheHeaders;
import org.eclipse.jgit.util.RawParseUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.PrintWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import javax.servlet.http.HttpServletResponse;
class BuckUtils {
private static final Logger log =
LoggerFactory.getLogger(BuckUtils.class);
static void build(Path root, Path gen, String target)
throws IOException, BuildFailureException {
log.info("buck build " + target);
Properties properties = loadBuckProperties(gen);
String buck = MoreObjects.firstNonNull(properties.getProperty("buck"), "buck");
ProcessBuilder proc = new ProcessBuilder(buck, "build", target)
.directory(root.toFile())
.redirectErrorStream(true);
if (properties.containsKey("PATH")) {
proc.environment().put("PATH", properties.getProperty("PATH"));
}
long start = TimeUtil.nowMs();
Process rebuild = proc.start();
byte[] out;
try (InputStream in = rebuild.getInputStream()) {
out = ByteStreams.toByteArray(in);
} finally {
rebuild.getOutputStream().close();
}
int status;
try {
status = rebuild.waitFor();
} catch (InterruptedException e) {
throw new InterruptedIOException("interrupted waiting for " + buck);
}
if (status != 0) {
throw new BuildFailureException(out);
}
long time = TimeUtil.nowMs() - start;
log.info(String.format("UPDATED %s in %.3fs", target, time / 1000.0));
}
private static Properties loadBuckProperties(Path gen)
throws FileNotFoundException, IOException {
Properties properties = new Properties();
try (InputStream in = new FileInputStream(
gen.resolve(Paths.get("tools/buck/buck.properties")).toFile())) {
properties.load(in);
}
return properties;
}
static void displayFailure(String rule, byte[] why, HttpServletResponse res)
throws IOException {
res.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
res.setContentType("text/html");
res.setCharacterEncoding(UTF_8.name());
CacheHeaders.setNotCacheable(res);
Escaper html = HtmlEscapers.htmlEscaper();
try (PrintWriter w = res.getWriter()) {
w.write("<html><title>BUILD FAILED</title><body>");
w.format("<h1>%s FAILED</h1>", html.escape(rule));
w.write("<pre>");
w.write(html.escape(RawParseUtils.decode(why)));
w.write("</pre>");
w.write("</body></html>");
}
}
static class BuildFailureException extends Exception {
private static final long serialVersionUID = 1L;
final byte[] why;
BuildFailureException(byte[] why) {
this.why = why;
}
}
}

74
gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/RebuildBowerComponentsFilter.java Normal file
View File

@ -0,0 +1,74 @@
// Copyright (C) 2015 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.httpd.raw;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.gerrit.httpd.raw.BuckUtils.BuildFailureException;
import com.google.gerrit.launcher.GerritLauncher;
import com.google.inject.Singleton;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletResponse;
@Singleton
class RebuildBowerComponentsFilter implements Filter {
private static final String TARGET = "//polygerrit-ui:polygerrit_components";
private final Path gen;
private final Path root;
private final Path zip;
RebuildBowerComponentsFilter(Path buckOut) {
gen = buckOut.resolve("gen");
root = buckOut.getParent();
zip = BowerComponentsServlet.getZipPath(buckOut);
}
@Override
public synchronized void doFilter(ServletRequest sreq, ServletResponse sres,
FilterChain chain) throws IOException, ServletException {
HttpServletResponse res = (HttpServletResponse) sres;
try {
BuckUtils.build(root, gen, TARGET);
} catch (BuildFailureException e) {
BuckUtils.displayFailure(TARGET, e.why, res);
return;
}
if (!Files.exists(zip)) {
String msg = "`buck build` did not produce " + zip.toAbsolutePath();
BuckUtils.displayFailure(TARGET, msg.getBytes(UTF_8), res);
}
GerritLauncher.reloadZipFileSystem(zip);
chain.doFilter(sreq, sres);
}
@Override
public void init(FilterConfig config) throws ServletException {
}
@Override
public void destroy() {
}
}

97
gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/RecompileGwtUiFilter.java
View File

@ -14,33 +14,16 @@
package com.google.gerrit.httpd.raw;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.base.MoreObjects;
import com.google.common.escape.Escaper;
import com.google.common.html.HtmlEscapers;
import com.google.common.io.ByteStreams;
import com.google.gerrit.common.TimeUtil;
import com.google.gerrit.httpd.raw.BuckUtils.BuildFailureException;
import com.google.gwtexpui.linker.server.UserAgentRule;
import com.google.gwtexpui.server.CacheHeaders;
import org.eclipse.jgit.util.RawParseUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.PrintWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@ -55,9 +38,6 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
class RecompileGwtUiFilter implements Filter {
private static final Logger log =
LoggerFactory.getLogger(RecompileGwtUiFilter.class);
private final boolean gwtuiRecompile =
System.getProperty("gerrit.disable-gwtui-recompile") == null;
private final UserAgentRule rule = new UserAgentRule();
@ -92,9 +72,9 @@ class RecompileGwtUiFilter implements Filter {
synchronized (this) {
try {
build(root, gen, rule);
BuckUtils.build(root, gen, rule);
} catch (BuildFailureException e) {
displayFailure(rule, e.why, (HttpServletResponse) res);
BuckUtils.displayFailure(rule, e.why, (HttpServletResponse) res);
return;
}
@ -109,24 +89,6 @@ class RecompileGwtUiFilter implements Filter {
chain.doFilter(request, res);
}
private void displayFailure(String rule, byte[] why, HttpServletResponse res)
throws IOException {
res.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
res.setContentType("text/html");
res.setCharacterEncoding(UTF_8.name());
CacheHeaders.setNotCacheable(res);
Escaper html = HtmlEscapers.htmlEscaper();
try (PrintWriter w = res.getWriter()) {
w.write("<html><title>BUILD FAILED</title><body>");
w.format("<h1>%s FAILED</h1>", html.escape(rule));
w.write("<pre>");
w.write(html.escape(RawParseUtils.decode(why)));
w.write("</pre>");
w.write("</body></html>");
}
}
@Override
public void init(FilterConfig config) {
}
@ -166,59 +128,6 @@ class RecompileGwtUiFilter implements Filter {
}
}
private static void build(Path root, Path gen, String target)
throws IOException, BuildFailureException {
log.info("buck build " + target);
Properties properties = loadBuckProperties(gen);
String buck = MoreObjects.firstNonNull(properties.getProperty("buck"), "buck");
ProcessBuilder proc = new ProcessBuilder(buck, "build", target)
.directory(root.toFile())
.redirectErrorStream(true);
if (properties.containsKey("PATH")) {
proc.environment().put("PATH", properties.getProperty("PATH"));
}
long start = TimeUtil.nowMs();
Process rebuild = proc.start();
byte[] out;
try (InputStream in = rebuild.getInputStream()) {
out = ByteStreams.toByteArray(in);
} finally {
rebuild.getOutputStream().close();
}
int status;
try {
status = rebuild.waitFor();
} catch (InterruptedException e) {
throw new InterruptedIOException("interrupted waiting for " + buck);
}
if (status != 0) {
throw new BuildFailureException(out);
}
long time = TimeUtil.nowMs() - start;
log.info(String.format("UPDATED %s in %.3fs", target, time / 1000.0));
}
private static Properties loadBuckProperties(Path gen)
throws FileNotFoundException, IOException {
Properties properties = new Properties();
try (InputStream in = new FileInputStream(
gen.resolve(Paths.get("tools/buck/buck.properties")).toFile())) {
properties.load(in);
}
return properties;
}
@SuppressWarnings("serial")
private static class BuildFailureException extends Exception {
final byte[] why;
BuildFailureException(byte[] why) {
this.why = why;
}
}
private static void mkdir(File dir) throws IOException {
if (!dir.isDirectory()) {
mkdir(dir.getParentFile());

5
gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java
View File

@ -108,8 +108,9 @@ public abstract class ResourceServlet extends HttpServlet {
*
* @param pathInfo result of {@link HttpServletRequest#getPathInfo()}.
* @return path where static content can be found.
* @throws IOException if an error occurred resolving the resource.
*/
protected abstract Path getResourcePath(String pathInfo);
protected abstract Path getResourcePath(String pathInfo) throws IOException;
protected FileTime getLastModifiedTime(Path p) throws IOException {
return Files.getLastModifiedTime(p);
@ -198,7 +199,7 @@ public abstract class ResourceServlet extends HttpServlet {
try {
Path p = getResourcePath(name);
return cache.get(p, newLoader(p));
} catch (ExecutionException e) {
} catch (ExecutionException | IOException e) {
log.warn(String.format("Cannot load static resource %s", name), e);
return null;
}

69
gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java
View File

@ -17,6 +17,7 @@ package com.google.gerrit.httpd.raw;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.cache.Cache;
import com.google.common.collect.ImmutableList;
import com.google.gerrit.httpd.GerritOptions;
import com.google.gerrit.httpd.raw.ResourceServlet.Resource;
import com.google.gerrit.launcher.GerritLauncher;
@ -39,10 +40,24 @@ import java.nio.file.Path;
import javax.servlet.http.HttpServlet;
public class StaticModule extends ServletModule {
private static final String GWT_UI_SERVLET = "GwtUiServlet";
private static final String BOWER_SERVLET = "BowerServlet";
public static final String CACHE = "static_content";
public static final ImmutableList<String> POLYGERRIT_INDEX_PATHS =
ImmutableList.of(
"/",
"/c/*",
"/q/*",
"/x/*",
"/admin/*",
"/dashboard/*",
"/settings/*",
// TODO(dborowitz): These fragments conflict with the REST API
// namespace, so they will need to use a different path.
"/groups/*",
"/projects/*");
private static final String GWT_UI_SERVLET = "GwtUiServlet";
private final GerritOptions options;
private Paths paths;
@ -104,21 +119,26 @@ public class StaticModule extends ServletModule {
private class PolyGerritUiModule extends ServletModule {
@Override
public void configureServlets() {
serve("/").with(PolyGerritUiIndexServlet.class);
serve("/c/*").with(PolyGerritUiIndexServlet.class);
serve("/q/*").with(PolyGerritUiIndexServlet.class);
serve("/x/*").with(PolyGerritUiIndexServlet.class);
serve("/admin/*").with(PolyGerritUiIndexServlet.class);
serve("/dashboard/*").with(PolyGerritUiIndexServlet.class);
serve("/settings/*").with(PolyGerritUiIndexServlet.class);
// TODO(dborowitz): These fragments conflict with the REST API namespace,
// so they will need to use a different path.
//serve("/groups/*").with(PolyGerritUiIndexServlet.class);
//serve("/projects/*").with(PolyGerritUiIndexServlet.class);
Path buckOut = getPaths().buckOut;
if (buckOut != null) {
RebuildBowerComponentsFilter rebuildFilter =
new RebuildBowerComponentsFilter(buckOut);
for (String p : POLYGERRIT_INDEX_PATHS) {
// Rebuilding bower_components once per load on the index request,
// is sufficient, since it will finish building before attempting to
// access any bower_components resources. Plus it saves contention and
// extraneous buck builds.
filter(p).through(rebuildFilter);
}
serve("/bower_components/*").with(BowerComponentsServlet.class);
} else {
// In the war case, bower_components are either inlined by vulcanize, or
// live under /polygerrit_ui in the war file, so we don't need a
// separate servlet.
}
if (getPaths().warFs == null) {
serve("/bower_components/*").with(
Key.get(PolyGerritUiServlet.class, Names.named(BOWER_SERVLET)));
for (String p : POLYGERRIT_INDEX_PATHS) {
serve(p).with(PolyGerritUiIndexServlet.class);
}
serve("/*").with(PolyGerritUiServlet.class);
}
@ -139,11 +159,9 @@ public class StaticModule extends ServletModule {
@Provides
@Singleton
@Named(BOWER_SERVLET)
PolyGerritUiServlet getPolyGerritUiBowerServlet(
BowerComponentsServlet getBowerComponentsServlet(
@Named(CACHE) Cache<Path, Resource> cache) {
return new PolyGerritUiServlet(cache,
polyGerritBasePath().resolveSibling("bower_components"));
return new BowerComponentsServlet(cache, getPaths().buckOut);
}
private Path polyGerritBasePath() {
@ -159,7 +177,7 @@ public class StaticModule extends ServletModule {
}
}
private static class Paths {
private class Paths {
private final FileSystem warFs;
private final Path buckOut;
private final Path unpackedWar;
@ -170,6 +188,9 @@ public class StaticModule extends ServletModule {
if (warFs == null) {
buckOut = getDeveloperBuckOut();
unpackedWar = makeWarTempDir();
} else if (options.forcePolyGerritDev()) {
buckOut = getDeveloperBuckOut();
unpackedWar = null;
} else {
buckOut = null;
unpackedWar = null;
@ -180,7 +201,7 @@ public class StaticModule extends ServletModule {
}
}
private static FileSystem getDistributionArchive() throws IOException {
private FileSystem getDistributionArchive() throws IOException {
File war;
try {
war = GerritLauncher.getDistributionArchive();
@ -198,7 +219,7 @@ public class StaticModule extends ServletModule {
return GerritLauncher.getZipFileSystem(war.toPath());
}
private static Path getDeveloperBuckOut() {
private Path getDeveloperBuckOut() {
try {
return GerritLauncher.getDeveloperBuckOut();
} catch (FileNotFoundException e) {
@ -206,7 +227,7 @@ public class StaticModule extends ServletModule {
}
}
private static Path makeWarTempDir() {
private Path makeWarTempDir() {
// Obtain our local temporary directory, but it comes back as a file
// so we have to switch it to be a directory post creation.
//

36
gerrit-launcher/src/main/java/com/google/gerrit/launcher/GerritLauncher.java
View File

@ -334,14 +334,44 @@ public final class GerritLauncher {
zip = zip.toRealPath();
FileSystem zipFs = zipFileSystems.get(zip);
if (zipFs == null) {
zipFs = FileSystems.newFileSystem(
URI.create("jar:" + zip.toUri()),
Collections.<String, String> emptyMap());
zipFs = newZipFileSystem(zip);
zipFileSystems.put(zip, zipFs);
}
return zipFs;
}
/**
* Reload the zip {@link FileSystem} for a path.
* <p>
* <strong>Warning</strong>: This calls {@link FileSystem#close()} on any
* previously open instance of the filesystem at this path, which may cause
* {@code IOException}s in any open path handles created with the old
* filesystem. Use with caution.
*
* @param zip path to zip file.
* @return reloaded filesystem instance.
* @throws IOException if there was an error reading the zip file.
*/
public static synchronized FileSystem reloadZipFileSystem(Path zip)
throws IOException {
// FileSystems canonicalizes the path, so we should too.
zip = zip.toRealPath();
@SuppressWarnings("resource") // Caching resource for later use.
FileSystem zipFs = zipFileSystems.get(zip);
if (zipFs != null) {
zipFs.close();
}
zipFs = newZipFileSystem(zip);
zipFileSystems.put(zip, zipFs);
return zipFs;
}
private static FileSystem newZipFileSystem(Path zip) throws IOException {
return FileSystems.newFileSystem(
URI.create("jar:" + zip.toUri()),
Collections.<String, String> emptyMap());
}
private static File locateMyArchive() throws FileNotFoundException {
final ClassLoader myCL = GerritLauncher.class.getClassLoader();
final String myName =