From 040d48af7045e75a7a08a15de4da2030e990f76b Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Tue, 3 Nov 2015 12:47:06 -0500 Subject: [PATCH 1/5] Refactor static content serving Extract an abstract base class from StaticServlet, called ResourceServlet, which knows how to read and cache static files from Paths. Implementations are responsible for translating incoming path info from the HttpServletRequest into URLs. Java 7 lets us register new FileSystems that read paths from a variety of locations, like zip files, so this will give us the flexibility to serve static paths from other locations in the future. Factor out a StaticModule from UrlModule for eventually handling all the various static URLs supported by Gerrit. Change-Id: I51f3fa11971959616aac6261d9f859f4eaeeaaf9 --- .../com/google/gerrit/httpd/UrlModule.java | 5 +- .../gerrit/httpd/raw/HostPageServlet.java | 2 +- .../gerrit/httpd/raw/ResourceServlet.java | 246 ++++++++++++++++++ .../google/gerrit/httpd/raw/StaticModule.java | 38 +++ .../gerrit/httpd/raw/StaticServlet.java | 216 ++------------- 5 files changed, 304 insertions(+), 203 deletions(-) create mode 100644 gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java create mode 100644 gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java index 86debdd2ce..7ecf2fa3e5 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/UrlModule.java @@ -23,7 +23,7 @@ import com.google.gerrit.httpd.raw.HostPageServlet; import com.google.gerrit.httpd.raw.LegacyGerritServlet; import com.google.gerrit.httpd.raw.RobotsServlet; import com.google.gerrit.httpd.raw.SshInfoServlet; -import com.google.gerrit.httpd.raw.StaticServlet; +import com.google.gerrit.httpd.raw.StaticModule; import com.google.gerrit.httpd.raw.ToolServlet; import com.google.gerrit.httpd.rpc.access.AccessRestApiServlet; import com.google.gerrit.httpd.rpc.account.AccountsRestApiServlet; @@ -77,7 +77,6 @@ class UrlModule extends ServletModule { serve("/signout").with(HttpLogoutServlet.class); } serve("/ssh_info").with(SshInfoServlet.class); - serve("/static/*").with(StaticServlet.class); serve("/Main.class").with(notFound()); serve("/com/google/gerrit/launcher/*").with(notFound()); @@ -107,6 +106,8 @@ class UrlModule extends ServletModule { filter("/Documentation/").through(QueryDocumentationFilter.class); serve("/robots.txt").with(RobotsServlet.class); + + install(new StaticModule()); } private Key notFound() { diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/HostPageServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/HostPageServlet.java index e21f973d79..57154393b4 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/HostPageServlet.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/HostPageServlet.java @@ -304,7 +304,7 @@ public class HostPageServlet extends HttpServlet { String src = e.getAttribute("src"); if (src != null && src.startsWith("static/")) { String name = src.substring("static/".length()); - StaticServlet.Resource r = staticServlet.getResource(name); + ResourceServlet.Resource r = staticServlet.getResource(name); if (r != null) { e.setAttribute("src", src + "?e=" + r.etag); } diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java new file mode 100644 index 0000000000..bb32ada520 --- /dev/null +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java @@ -0,0 +1,246 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.httpd.raw; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.net.HttpHeaders.CONTENT_ENCODING; +import static com.google.common.net.HttpHeaders.ETAG; +import static com.google.common.net.HttpHeaders.IF_NONE_MATCH; +import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; +import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED; + +import com.google.common.base.CharMatcher; +import com.google.common.cache.Cache; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.gerrit.common.Nullable; +import com.google.gerrit.httpd.HtmlDomUtil; +import com.google.gwtexpui.server.CacheHeaders; +import com.google.gwtjsonrpc.server.RPCServletUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Base class for serving static resources. + *

+ * Supports caching, ETags, basic content type detection, and limited gzip + * compression. + */ +public abstract class ResourceServlet extends HttpServlet { + private static final long serialVersionUID = 1L; + + private static final Logger log = + LoggerFactory.getLogger(ResourceServlet.class); + + private static final String JS = "application/x-javascript"; + private static final ImmutableMap MIME_TYPES = + ImmutableMap. builder() + .put("css", "text/css") + .put("gif", "image/gif") + .put("htm", "text/html") + .put("html", "text/html") + .put("jpeg", "image/jpeg") + .put("jpg", "image/jpeg") + .put("js", JS) + .put("pdf", "application/pdf") + .put("png", "image/png") + .put("rtf", "text/rtf") + .put("svg", "image/svg+xml") + .put("text", "text/plain") + .put("tif", "image/tiff") + .put("tiff", "image/tiff") + .put("txt", "text/plain") + .build(); + + protected static String contentType(String name) { + int dot = name.lastIndexOf('.'); + String ext = 0 < dot ? name.substring(dot + 1) : ""; + String type = MIME_TYPES.get(ext); + return type != null ? type : "application/octet-stream"; + } + + private final Cache cache; + private final boolean refresh; + + protected ResourceServlet(Cache cache, boolean refresh) { + this.cache = checkNotNull(cache, "cache"); + this.refresh = refresh; + } + + /** + * Get the resource path on the filesystem that should be served for this + * request. + * + * @param pathInfo result of {@link HttpServletRequest#getPathInfo()}. + * @return path where static content can be found. + */ + protected abstract Path getResourcePath(String pathInfo); + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse rsp) + throws IOException { + Resource r; + try { + r = getResource(req); + } catch (ExecutionException e) { + log.warn(String.format( + "Cannot load static resource %s", + req.getPathInfo()), e); + CacheHeaders.setNotCacheable(rsp); + rsp.setStatus(SC_INTERNAL_SERVER_ERROR); + return; + } + + String e = req.getParameter("e"); + if (r == Resource.NOT_FOUND || (e != null && !r.etag.equals(e))) { + CacheHeaders.setNotCacheable(rsp); + rsp.setStatus(SC_NOT_FOUND); + return; + } else if (r.etag.equals(req.getHeader(IF_NONE_MATCH))) { + rsp.setStatus(SC_NOT_MODIFIED); + return; + } + + byte[] tosend = r.raw; + if (!r.contentType.equals(JS) && RPCServletUtils.acceptsGzipEncoding(req)) { + byte[] gz = HtmlDomUtil.compress(tosend); + if ((gz.length + 24) < tosend.length) { + rsp.setHeader(CONTENT_ENCODING, "gzip"); + tosend = gz; + } + } + if (e != null && r.etag.equals(e)) { + CacheHeaders.setCacheable(req, rsp, 360, DAYS, false); + } else { + CacheHeaders.setCacheable(req, rsp, 15, MINUTES, refresh); + } + rsp.setHeader(ETAG, r.etag); + rsp.setContentType(r.contentType); + rsp.setContentLength(tosend.length); + try (OutputStream out = rsp.getOutputStream()) { + out.write(tosend); + } + } + + @Nullable + Resource getResource(String name) { + try { + Path p = getResourcePath(name); + return cache.get(p, newLoader(p)); + } catch (ExecutionException e) { + log.warn(String.format("Cannot load static resource %s", name), e); + return null; + } + } + + private Resource getResource(HttpServletRequest req) + throws ExecutionException { + String name = CharMatcher.is('/').trimFrom(req.getPathInfo()); + if (isUnreasonableName(name)) { + return Resource.NOT_FOUND; + } + Path p = getResourcePath(name); + if (p == null) { + return Resource.NOT_FOUND; + } + + Callable loader = newLoader(p); + Resource r = cache.get(p, loader); + if (r == Resource.NOT_FOUND) { + return Resource.NOT_FOUND; + } + + if (refresh && r.isStale(p)) { + cache.invalidate(p); + r = cache.get(p, loader); + } + return r; + } + + private static boolean isUnreasonableName(String name) { + return name.length() < 1 + || name.contains("\\") // no windows/dos style paths + || name.startsWith("../") // no "../etc/passwd" + || name.contains("/../") // no "foo/../etc/passwd" + || name.contains("/./") // "foo/./foo" is insane to ask + || name.contains("//"); // windows UNC path can be "//..." + } + + private Callable newLoader(final Path p) { + return new Callable() { + @Override + public Resource call() throws IOException { + try { + return new Resource( + Files.getLastModifiedTime(p), + contentType(p.toString()), + Files.readAllBytes(p)); + } catch (FileNotFoundException e) { + return Resource.NOT_FOUND; + } + } + }; + } + + static class Resource { + static final Resource NOT_FOUND = + new Resource(FileTime.fromMillis(0), "", new byte[] {}); + + final FileTime lastModified; + final String contentType; + final String etag; + final byte[] raw; + + Resource(FileTime lastModified, String contentType, byte[] raw) { + this.lastModified = checkNotNull(lastModified, "lastModified"); + this.contentType = checkNotNull(contentType, "contentType"); + this.raw = checkNotNull(raw, "raw"); + this.etag = Hashing.md5().hashBytes(raw).toString(); + } + + boolean isStale(Path p) { + try { + return !lastModified.equals(Files.getLastModifiedTime(p)); + } catch (IOException e) { + return true; + } + } + } + + static class Weigher + implements com.google.common.cache.Weigher { + @Override + public int weigh(Path p, Resource r) { + return 2 * p.toString().length() + r.raw.length; + } + } +} diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java new file mode 100644 index 0000000000..258e09a549 --- /dev/null +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java @@ -0,0 +1,38 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.httpd.raw; + +import com.google.gerrit.httpd.raw.ResourceServlet.Resource; +import com.google.gerrit.server.cache.CacheModule; +import com.google.inject.servlet.ServletModule; + +import java.nio.file.Path; + +public class StaticModule extends ServletModule { + static final String CACHE = "static_content"; + + @Override + protected void configureServlets() { + serve("/static/*").with(StaticServlet.class); + install(new CacheModule() { + @Override + protected void configure() { + cache(CACHE, Path.class, Resource.class) + .maximumWeight(1 << 20) + .weigher(ResourceServlet.Weigher.class); + } + }); + } +} diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticServlet.java index 570ad57df1..8e290c9314 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticServlet.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticServlet.java @@ -14,88 +14,32 @@ package com.google.gerrit.httpd.raw; -import static com.google.common.net.HttpHeaders.CONTENT_ENCODING; -import static com.google.common.net.HttpHeaders.ETAG; -import static com.google.common.net.HttpHeaders.IF_NONE_MATCH; -import static com.google.gerrit.common.FileUtil.lastModified; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; -import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; -import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED; - -import com.google.common.base.CharMatcher; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.Weigher; -import com.google.common.collect.Maps; -import com.google.common.hash.Hashing; -import com.google.gerrit.common.FileUtil; -import com.google.gerrit.common.Nullable; -import com.google.gerrit.httpd.HtmlDomUtil; +import com.google.common.cache.Cache; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; -import com.google.gwtexpui.server.CacheHeaders; -import com.google.gwtjsonrpc.server.RPCServletUtils; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.google.inject.name.Named; import org.eclipse.jgit.lib.Config; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.util.Map; -import java.util.concurrent.ExecutionException; - -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; /** Sends static content from the site 's {@code static/} subdirectory. */ -@SuppressWarnings("serial") @Singleton -public class StaticServlet extends HttpServlet { - private static final Logger log = LoggerFactory.getLogger(StaticServlet.class); - private static final String JS = "application/x-javascript"; - private static final Map MIME_TYPES = Maps.newHashMap(); - static { - MIME_TYPES.put("html", "text/html"); - MIME_TYPES.put("htm", "text/html"); - MIME_TYPES.put("js", JS); - MIME_TYPES.put("css", "text/css"); - MIME_TYPES.put("rtf", "text/rtf"); - MIME_TYPES.put("txt", "text/plain"); - MIME_TYPES.put("text", "text/plain"); - MIME_TYPES.put("pdf", "application/pdf"); - MIME_TYPES.put("jpeg", "image/jpeg"); - MIME_TYPES.put("jpg", "image/jpeg"); - MIME_TYPES.put("gif", "image/gif"); - MIME_TYPES.put("png", "image/png"); - MIME_TYPES.put("tiff", "image/tiff"); - MIME_TYPES.put("tif", "image/tiff"); - MIME_TYPES.put("svg", "image/svg+xml"); - } - - private static String contentType(final String name) { - final int dot = name.lastIndexOf('.'); - final String ext = 0 < dot ? name.substring(dot + 1) : ""; - final String type = MIME_TYPES.get(ext); - return type != null ? type : "application/octet-stream"; - } +public class StaticServlet extends ResourceServlet { + private static final long serialVersionUID = 1L; private final Path staticBase; - private final boolean refresh; - private final LoadingCache cache; @Inject - StaticServlet(@GerritServerConfig Config cfg, SitePaths site) { + StaticServlet( + SitePaths site, + @GerritServerConfig Config cfg, + @Named(StaticModule.CACHE) Cache cache) { + super(cache, cfg.getBoolean("site", "refreshHeaderFooter", true)); Path p; try { p = site.static_dir.toRealPath().normalize(); @@ -103,147 +47,19 @@ public class StaticServlet extends HttpServlet { p = site.static_dir.toAbsolutePath().normalize(); } staticBase = p; - refresh = cfg.getBoolean("site", "refreshHeaderFooter", true); - cache = CacheBuilder.newBuilder() - .maximumWeight(1 << 20) - .weigher(new Weigher() { - @Override - public int weigh(String name, Resource r) { - return 2 * name.length() + r.raw.length; - } - }) - .build(new CacheLoader() { - @Override - public Resource load(String name) throws Exception { - return loadResource(name); - } - }); - } - - @Nullable - Resource getResource(String name) { - try { - return cache.get(name); - } catch (ExecutionException e) { - log.warn(String.format("Cannot load static resource %s", name), e); - return null; - } - } - - private Resource getResource(HttpServletRequest req) - throws ExecutionException { - String name = CharMatcher.is('/').trimFrom(req.getPathInfo()); - if (isUnreasonableName(name)) { - return Resource.NOT_FOUND; - } - - Resource r = cache.get(name); - if (r == Resource.NOT_FOUND) { - return Resource.NOT_FOUND; - } - - if (refresh && r.isStale()) { - cache.invalidate(name); - r = cache.get(name); - } - return r; - } - - private static boolean isUnreasonableName(String name) { - return name.length() < 1 - || name.contains("\\") // no windows/dos style paths - || name.startsWith("../") // no "../etc/passwd" - || name.contains("/../") // no "foo/../etc/passwd" - || name.contains("/./") // "foo/./foo" is insane to ask - || name.contains("//"); // windows UNC path can be "//..." } @Override - protected void doGet(final HttpServletRequest req, - final HttpServletResponse rsp) throws IOException { - Resource r; - try { - r = getResource(req); - } catch (ExecutionException e) { - log.warn(String.format( - "Cannot load static resource %s", - req.getPathInfo()), e); - CacheHeaders.setNotCacheable(rsp); - rsp.setStatus(SC_INTERNAL_SERVER_ERROR); - return; - } - - String e = req.getParameter("e"); - if (r == Resource.NOT_FOUND || (e != null && !r.etag.equals(e))) { - CacheHeaders.setNotCacheable(rsp); - rsp.setStatus(SC_NOT_FOUND); - return; - } else if (r.etag.equals(req.getHeader(IF_NONE_MATCH))) { - rsp.setStatus(SC_NOT_MODIFIED); - return; - } - - byte[] tosend = r.raw; - if (!r.contentType.equals(JS) && RPCServletUtils.acceptsGzipEncoding(req)) { - byte[] gz = HtmlDomUtil.compress(tosend); - if ((gz.length + 24) < tosend.length) { - rsp.setHeader(CONTENT_ENCODING, "gzip"); - tosend = gz; - } - } - if (e != null && r.etag.equals(e)) { - CacheHeaders.setCacheable(req, rsp, 360, DAYS, false); - } else { - CacheHeaders.setCacheable(req, rsp, 15, MINUTES, refresh); - } - rsp.setHeader(ETAG, r.etag); - rsp.setContentType(r.contentType); - rsp.setContentLength(tosend.length); - try (OutputStream out = rsp.getOutputStream()) { - out.write(tosend); - } - } - - private Resource loadResource(String name) throws IOException { - Path p = staticBase.resolve(name); + protected Path getResourcePath(String pathInfo) { + Path p = staticBase.resolve(pathInfo); try { p = p.toRealPath().normalize(); + if (!p.startsWith(staticBase)) { + return null; + } + return p; } catch (IOException e) { - return Resource.NOT_FOUND; - } - if (!p.startsWith(staticBase)) { - return Resource.NOT_FOUND; - } - - long ts = FileUtil.lastModified(p); - byte[] raw; - try { - raw = Files.readAllBytes(p); - } catch (NoSuchFileException e) { - return Resource.NOT_FOUND; - } - return new Resource(p, ts, contentType(name), raw); - } - - static class Resource { - static final Resource NOT_FOUND = new Resource(null, -1, "", new byte[] {}); - - final Path src; - final long lastModified; - final String contentType; - final String etag; - final byte[] raw; - - Resource(Path src, long lastModified, String contentType, byte[] raw) { - this.src = src; - this.lastModified = lastModified; - this.contentType = contentType; - this.etag = Hashing.md5().hashBytes(raw).toString(); - this.raw = raw; - } - - boolean isStale() { - return lastModified != lastModified(src); + return null; } } } From 412678ac886356e4516967bce735ed0797f00214 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Tue, 3 Nov 2015 13:13:46 -0500 Subject: [PATCH 2/5] Rename StaticServlet to SiteStaticDirectoryServlet "Static servlet' sounds like it could serve any kind of static content, but this is a very specific kind of static content: files stored in the static/ directory under a Gerrit site. Change-Id: I2c123e32d15c9750617a4377d7d443e48e0e1e30 --- .../java/com/google/gerrit/httpd/raw/HostPageServlet.java | 4 ++-- .../{StaticServlet.java => SiteStaticDirectoryServlet.java} | 4 ++-- .../main/java/com/google/gerrit/httpd/raw/StaticModule.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/{StaticServlet.java => SiteStaticDirectoryServlet.java} (95%) diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/HostPageServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/HostPageServlet.java index 57154393b4..76d2cdb327 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/HostPageServlet.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/HostPageServlet.java @@ -93,7 +93,7 @@ public class HostPageServlet extends HttpServlet { private final Document template; private final String noCacheName; private final boolean refreshHeaderFooter; - private final StaticServlet staticServlet; + private final SiteStaticDirectoryServlet staticServlet; private final boolean isNoteDbEnabled; private final Integer pluginsLoadTimeout; private final GetDiffPreferences getDiff; @@ -109,7 +109,7 @@ public class HostPageServlet extends HttpServlet { DynamicSet webUiPlugins, DynamicSet motd, @GerritServerConfig Config cfg, - StaticServlet ss, + SiteStaticDirectoryServlet ss, NotesMigration migration, GetDiffPreferences diffPref) throws IOException, ServletException { diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/SiteStaticDirectoryServlet.java similarity index 95% rename from gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticServlet.java rename to gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/SiteStaticDirectoryServlet.java index 8e290c9314..cf99d3c645 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticServlet.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/SiteStaticDirectoryServlet.java @@ -29,13 +29,13 @@ import java.nio.file.Path; /** Sends static content from the site 's {@code static/} subdirectory. */ @Singleton -public class StaticServlet extends ResourceServlet { +public class SiteStaticDirectoryServlet extends ResourceServlet { private static final long serialVersionUID = 1L; private final Path staticBase; @Inject - StaticServlet( + SiteStaticDirectoryServlet( SitePaths site, @GerritServerConfig Config cfg, @Named(StaticModule.CACHE) Cache cache) { diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java index 258e09a549..12fe177629 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java @@ -25,7 +25,7 @@ public class StaticModule extends ServletModule { @Override protected void configureServlets() { - serve("/static/*").with(StaticServlet.class); + serve("/static/*").with(SiteStaticDirectoryServlet.class); install(new CacheModule() { @Override protected void configure() { From 712669ac36b267d4ab8558f740311c4bdb9f9788 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Tue, 3 Nov 2015 14:25:04 -0500 Subject: [PATCH 3/5] ResourceServlet: Respect existing cache headers This servlet sits behind CacheControlFilter, which may set cache headers (e.g. to disable caching for *.nocache.js). If headers were set, don't touch them from this servlet. This is a safer change than the alternative of moving CacheControlFilter's action to after the filter chain. Change-Id: I8920c48e9c015cc34b9b9228abc773fd4f59c547 --- .../java/com/google/gwtexpui/server/CacheHeaders.java | 6 ++++++ .../com/google/gerrit/httpd/raw/ResourceServlet.java | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/gerrit-gwtexpui/src/main/java/com/google/gwtexpui/server/CacheHeaders.java b/gerrit-gwtexpui/src/main/java/com/google/gwtexpui/server/CacheHeaders.java index 4b789918ae..950400acd9 100644 --- a/gerrit-gwtexpui/src/main/java/com/google/gwtexpui/server/CacheHeaders.java +++ b/gerrit-gwtexpui/src/main/java/com/google/gwtexpui/server/CacheHeaders.java @@ -128,6 +128,12 @@ public class CacheHeaders { cache(res, "private", age, unit, mustRevalidate); } + public static boolean hasCacheHeader(HttpServletResponse res) { + return res.getHeader("Cache-Control") != null + || res.getHeader("Expires") != null + || "no-cache".equals(res.getHeader("Pragma")); + } + private static void cache(HttpServletResponse res, String type, long age, TimeUnit unit, boolean revalidate) { res.setHeader("Cache-Control", String.format( diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java index bb32ada520..50b1c2e484 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java @@ -138,10 +138,12 @@ public abstract class ResourceServlet extends HttpServlet { tosend = gz; } } - if (e != null && r.etag.equals(e)) { - CacheHeaders.setCacheable(req, rsp, 360, DAYS, false); - } else { - CacheHeaders.setCacheable(req, rsp, 15, MINUTES, refresh); + if (!CacheHeaders.hasCacheHeader(rsp)) { + if (e != null && r.etag.equals(e)) { + CacheHeaders.setCacheable(req, rsp, 360, DAYS, false); + } else { + CacheHeaders.setCacheable(req, rsp, 15, MINUTES, refresh); + } } rsp.setHeader(ETAG, r.etag); rsp.setContentType(r.contentType); From 74317d4a076c60016c2e2284eb88b86ce59e7232 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Tue, 3 Nov 2015 16:11:00 -0500 Subject: [PATCH 4/5] ResourceServlet: Stream large files, bypassing the cache Sufficiently large static files will thrash the cache. Don't bother even storing them in the cache; stream them directly from their Paths. Instead of using a content-based hash for the ETag, just use Last-Modified based on the modified time of the file. Change-Id: Idc9b5daf9f5da124eb0d6b8b21806866de3d3a6b --- .../gerrit/httpd/raw/ResourceServlet.java | 131 +++++++++++++----- 1 file changed, 97 insertions(+), 34 deletions(-) diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java index 50b1c2e484..e6ad1b3795 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java @@ -17,7 +17,9 @@ package com.google.gerrit.httpd.raw; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.net.HttpHeaders.CONTENT_ENCODING; import static com.google.common.net.HttpHeaders.ETAG; +import static com.google.common.net.HttpHeaders.IF_MODIFIED_SINCE; import static com.google.common.net.HttpHeaders.IF_NONE_MATCH; +import static com.google.common.net.HttpHeaders.LAST_MODIFIED; import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.MINUTES; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; @@ -28,6 +30,7 @@ import com.google.common.base.CharMatcher; import com.google.common.cache.Cache; import com.google.common.collect.ImmutableMap; import com.google.common.hash.Hashing; +import com.google.gerrit.common.FileUtil; import com.google.gerrit.common.Nullable; import com.google.gerrit.httpd.HtmlDomUtil; import com.google.gwtexpui.server.CacheHeaders; @@ -36,14 +39,15 @@ import com.google.gwtjsonrpc.server.RPCServletUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.attribute.FileTime; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.zip.GZIPOutputStream; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; @@ -61,6 +65,8 @@ public abstract class ResourceServlet extends HttpServlet { private static final Logger log = LoggerFactory.getLogger(ResourceServlet.class); + private static final int CACHE_FILE_SIZE_LIMIT_BYTES = 100 << 10; + private static final String JS = "application/x-javascript"; private static final ImmutableMap MIME_TYPES = ImmutableMap. builder() @@ -108,20 +114,45 @@ public abstract class ResourceServlet extends HttpServlet { @Override protected void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - Resource r; - try { - r = getResource(req); - } catch (ExecutionException e) { - log.warn(String.format( - "Cannot load static resource %s", - req.getPathInfo()), e); - CacheHeaders.setNotCacheable(rsp); - rsp.setStatus(SC_INTERNAL_SERVER_ERROR); + String name = CharMatcher.is('/').trimFrom(req.getPathInfo()); + if (isUnreasonableName(name)) { + notFound(rsp); + return; + } + Path p = getResourcePath(name); + if (p == null) { + notFound(rsp); + return; + } + + Resource r = cache.getIfPresent(p); + if (r == null && maybeStream(p, req, rsp)) { + return; + } + + if (r == null) { + Callable loader = newLoader(p); + try { + r = cache.get(p, loader); + if (refresh && r.isStale(p)) { + cache.invalidate(p); + r = cache.get(p, loader); + } + } catch (ExecutionException e) { + log.warn("Cannot load static resource " + req.getPathInfo(), e); + CacheHeaders.setNotCacheable(rsp); + rsp.setStatus(SC_INTERNAL_SERVER_ERROR); + return; + } + } + + if (r == Resource.NOT_FOUND) { + notFound(rsp); return; } String e = req.getParameter("e"); - if (r == Resource.NOT_FOUND || (e != null && !r.etag.equals(e))) { + if (e != null && !r.etag.equals(e)) { CacheHeaders.setNotCacheable(rsp); rsp.setStatus(SC_NOT_FOUND); return; @@ -164,30 +195,62 @@ public abstract class ResourceServlet extends HttpServlet { } } - private Resource getResource(HttpServletRequest req) - throws ExecutionException { - String name = CharMatcher.is('/').trimFrom(req.getPathInfo()); - if (isUnreasonableName(name)) { - return Resource.NOT_FOUND; - } - Path p = getResourcePath(name); - if (p == null) { - return Resource.NOT_FOUND; - } - - Callable loader = newLoader(p); - Resource r = cache.get(p, loader); - if (r == Resource.NOT_FOUND) { - return Resource.NOT_FOUND; - } - - if (refresh && r.isStale(p)) { - cache.invalidate(p); - r = cache.get(p, loader); - } - return r; + private static void notFound(HttpServletResponse rsp) { + rsp.setStatus(SC_NOT_FOUND); + CacheHeaders.setNotCacheable(rsp); } + /** + * Maybe stream a path to the response, depending on the properties of the + * file and cache headers in the request. + * + * @param p path to stream + * @param req HTTP request. + * @param rsp HTTP response. + * @return true if the response was written (either the file contents or an + * error); false if the path is too small to stream and should be cached. + */ + private boolean maybeStream(Path p, HttpServletRequest req, + HttpServletResponse rsp) throws IOException { + try { + if (Files.size(p) < CACHE_FILE_SIZE_LIMIT_BYTES) { + return false; + } + } catch (NoSuchFileException e) { + cache.put(p, Resource.NOT_FOUND); + notFound(rsp); + return true; + } + + long lastModified = FileUtil.lastModified(p); + if (req.getDateHeader(IF_MODIFIED_SINCE) >= lastModified) { + rsp.setStatus(SC_NOT_MODIFIED); + return true; + } + + if (lastModified > 0) { + rsp.setDateHeader(LAST_MODIFIED, lastModified); + } + if (!CacheHeaders.hasCacheHeader(rsp)) { + CacheHeaders.setCacheable(req, rsp, 15, MINUTES, refresh); + } + rsp.setContentType(contentType(p.toString())); + + OutputStream out = rsp.getOutputStream(); + GZIPOutputStream gz = null; + if (RPCServletUtils.acceptsGzipEncoding(req)) { + rsp.setHeader(CONTENT_ENCODING, "gzip"); + gz = new GZIPOutputStream(out); + out = gz; + } + Files.copy(p, out); + if (gz != null) { + gz.finish(); + } + return true; + } + + private static boolean isUnreasonableName(String name) { return name.length() < 1 || name.contains("\\") // no windows/dos style paths @@ -206,7 +269,7 @@ public abstract class ResourceServlet extends HttpServlet { Files.getLastModifiedTime(p), contentType(p.toString()), Files.readAllBytes(p)); - } catch (FileNotFoundException e) { + } catch (NoSuchFileException e) { return Resource.NOT_FOUND; } } From c916b9e6a91864024185b820ccede36546805794 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Tue, 3 Nov 2015 13:12:41 -0500 Subject: [PATCH 5/5] Serve GWT UI from ResourceServlet We already have a somewhat-featureful static content servlet for serving data from /static; use it for the GWT UI as well. Java's zip filesystem support makes the war case easy; we don't have to do the extract-to-a-directory hack that makes it work with Jetty. The developer case is also pretty easy, though we have to move the filter to recompile the GWT UI into the httpd package. One other wrinkle is that the GWT build process puts bogus timestamps on the GWT compiler output, so we need to pretend the timestamps on all the files are the startup time of the server. This means clients will have to re-download large identical JS assets after a server restart even if the assets didn't change. Gerrit has mostly pretty good uptime so this is not a huge deal. Change-Id: I0a7ade3cadf3a4a4e1726b56b87b0cbe4c6e0c93 --- gerrit-httpd/BUCK | 2 + .../httpd/raw/DeveloperGwtUiServlet.java | 51 ++++ .../httpd/raw/RecompileGwtUiFilter.java | 231 +++++++++++++++ .../gerrit/httpd/raw/ResourceServlet.java | 21 +- .../google/gerrit/httpd/raw/StaticModule.java | 101 +++++++ .../gerrit/httpd/raw/WarGwtUiServlet.java | 47 +++ gerrit-launcher/BUCK | 1 + .../gerrit/launcher/GerritLauncher.java | 31 +- .../gerrit/pgm/http/jetty/JettyServer.java | 267 +----------------- 9 files changed, 475 insertions(+), 277 deletions(-) create mode 100644 gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/DeveloperGwtUiServlet.java create mode 100644 gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/RecompileGwtUiFilter.java create mode 100644 gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/WarGwtUiServlet.java diff --git a/gerrit-httpd/BUCK b/gerrit-httpd/BUCK index b29bd2ac48..da224d5d01 100644 --- a/gerrit-httpd/BUCK +++ b/gerrit-httpd/BUCK @@ -12,7 +12,9 @@ java_library( '//gerrit-common:annotations', '//gerrit-common:server', '//gerrit-extension-api:api', + '//gerrit-gwtexpui:linker_server', '//gerrit-gwtexpui:server', + '//gerrit-launcher:launcher', '//gerrit-patch-jgit:server', '//gerrit-prettify:server', '//gerrit-reviewdb:server', diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/DeveloperGwtUiServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/DeveloperGwtUiServlet.java new file mode 100644 index 0000000000..9d57b17b57 --- /dev/null +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/DeveloperGwtUiServlet.java @@ -0,0 +1,51 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.httpd.raw; + +import com.google.common.cache.Cache; +import com.google.gerrit.common.TimeUtil; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; + +class DeveloperGwtUiServlet extends ResourceServlet { + private static final long serialVersionUID = 1L; + + private static final FileTime NOW = FileTime.fromMillis(TimeUtil.nowMs()); + + private final Path ui; + + DeveloperGwtUiServlet(Cache cache, Path unpackedWar) + throws IOException { + super(cache, false); + ui = unpackedWar.resolve("gerrit_ui"); + Files.createDirectory(ui); + ui.toFile().deleteOnExit(); + } + + @Override + protected Path getResourcePath(String pathInfo) { + return ui.resolve(pathInfo); + } + + @Override + protected FileTime getLastModifiedTime(Path p) { + // Return initialization time of this class, since the GWT outputs from the + // build process all have mtimes of 1980/1/1. + return NOW; + } +} diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/RecompileGwtUiFilter.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/RecompileGwtUiFilter.java new file mode 100644 index 0000000000..a5bc6c661f --- /dev/null +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/RecompileGwtUiFilter.java @@ -0,0 +1,231 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.httpd.raw; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.MoreObjects; +import com.google.common.escape.Escaper; +import com.google.common.html.HtmlEscapers; +import com.google.common.io.ByteStreams; +import com.google.gerrit.common.TimeUtil; +import com.google.gwtexpui.linker.server.UserAgentRule; +import com.google.gwtexpui.server.CacheHeaders; + +import org.eclipse.jgit.util.RawParseUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.PrintWriter; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +class RecompileGwtUiFilter implements Filter { + private static final Logger log = + LoggerFactory.getLogger(RecompileGwtUiFilter.class); + + private final boolean gwtuiRecompile = + System.getProperty("gerrit.disable-gwtui-recompile") == null; + private final UserAgentRule rule = new UserAgentRule(); + private final Set uaInitialized = new HashSet<>(); + private final Path unpackedWar; + private final Path gen; + private final Path root; + + private String lastTarget; + private long lastTime; + + RecompileGwtUiFilter(Path buckOut, Path unpackedWar) { + this.unpackedWar = unpackedWar; + gen = buckOut.resolve("gen"); + root = buckOut.getParent(); + } + + @Override + public void doFilter(ServletRequest request, ServletResponse res, + FilterChain chain) throws IOException, ServletException { + String pkg = "gerrit-gwtui"; + String target = "ui_" + rule.select((HttpServletRequest) request); + if (gwtuiRecompile || !uaInitialized.contains(target)) { + String rule = "//" + pkg + ":" + target; + // TODO(davido): instead of assuming specific Buck's internal + // target directory for gwt_binary() artifacts, ask Buck for + // the location of user agent permutation GWT zip, e. g.: + // $ buck targets --show_output //gerrit-gwtui:ui_safari \ + // | awk '{print $2}' + String child = String.format("%s/__gwt_binary_%s__", pkg, target); + File zip = gen.resolve(child).resolve(target + ".zip").toFile(); + + synchronized (this) { + try { + build(root, gen, rule); + } catch (BuildFailureException e) { + displayFailure(rule, e.why, (HttpServletResponse) res); + return; + } + + if (!target.equals(lastTarget) || lastTime != zip.lastModified()) { + lastTarget = target; + lastTime = zip.lastModified(); + unpack(zip, unpackedWar.toFile()); + } + } + uaInitialized.add(target); + } + chain.doFilter(request, res); + } + + private void displayFailure(String rule, byte[] why, HttpServletResponse res) + throws IOException { + res.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + res.setContentType("text/html"); + res.setCharacterEncoding(UTF_8.name()); + CacheHeaders.setNotCacheable(res); + + Escaper html = HtmlEscapers.htmlEscaper(); + try (PrintWriter w = res.getWriter()) { + w.write("BUILD FAILED"); + w.format("

%s FAILED

", html.escape(rule)); + w.write("
");
+      w.write(html.escape(RawParseUtils.decode(why)));
+      w.write("
"); + w.write(""); + } + } + + @Override + public void init(FilterConfig config) { + } + + @Override + public void destroy() { + } + + private static void unpack(File srcwar, File dstwar) throws IOException { + try (ZipFile zf = new ZipFile(srcwar)) { + final Enumeration e = zf.entries(); + while (e.hasMoreElements()) { + final ZipEntry ze = e.nextElement(); + final String name = ze.getName(); + + if (ze.isDirectory() + || name.startsWith("WEB-INF/") + || name.startsWith("META-INF/") + || name.startsWith("com/google/gerrit/launcher/") + || name.equals("Main.class")) { + continue; + } + + final File rawtmp = new File(dstwar, name); + mkdir(rawtmp.getParentFile()); + rawtmp.deleteOnExit(); + + try (FileOutputStream rawout = new FileOutputStream(rawtmp); + InputStream in = zf.getInputStream(ze)) { + final byte[] buf = new byte[4096]; + int n; + while ((n = in.read(buf, 0, buf.length)) > 0) { + rawout.write(buf, 0, n); + } + } + } + } + } + + private static void build(Path root, Path gen, String target) + throws IOException, BuildFailureException { + log.info("buck build " + target); + Properties properties = loadBuckProperties(gen); + String buck = MoreObjects.firstNonNull(properties.getProperty("buck"), "buck"); + ProcessBuilder proc = new ProcessBuilder(buck, "build", target) + .directory(root.toFile()) + .redirectErrorStream(true); + if (properties.containsKey("PATH")) { + proc.environment().put("PATH", properties.getProperty("PATH")); + } + long start = TimeUtil.nowMs(); + Process rebuild = proc.start(); + byte[] out; + try (InputStream in = rebuild.getInputStream()) { + out = ByteStreams.toByteArray(in); + } finally { + rebuild.getOutputStream().close(); + } + + int status; + try { + status = rebuild.waitFor(); + } catch (InterruptedException e) { + throw new InterruptedIOException("interrupted waiting for " + buck); + } + if (status != 0) { + throw new BuildFailureException(out); + } + + long time = TimeUtil.nowMs() - start; + log.info(String.format("UPDATED %s in %.3fs", target, time / 1000.0)); + } + + private static Properties loadBuckProperties(Path gen) + throws FileNotFoundException, IOException { + Properties properties = new Properties(); + try (InputStream in = new FileInputStream( + gen.resolve(Paths.get("tools/buck/buck.properties")).toFile())) { + properties.load(in); + } + return properties; + } + + @SuppressWarnings("serial") + private static class BuildFailureException extends Exception { + final byte[] why; + + BuildFailureException(byte[] why) { + this.why = why; + } + } + + private static void mkdir(File dir) throws IOException { + if (!dir.isDirectory()) { + mkdir(dir.getParentFile()); + if (!dir.mkdir()) { + throw new IOException("Cannot mkdir " + dir.getAbsolutePath()); + } + dir.deleteOnExit(); + } + } +} diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java index e6ad1b3795..a804e2a5d7 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/ResourceServlet.java @@ -111,6 +111,10 @@ public abstract class ResourceServlet extends HttpServlet { */ protected abstract Path getResourcePath(String pathInfo); + protected FileTime getLastModifiedTime(Path p) throws IOException { + return Files.getLastModifiedTime(p); + } + @Override protected void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException { @@ -134,11 +138,11 @@ public abstract class ResourceServlet extends HttpServlet { Callable loader = newLoader(p); try { r = cache.get(p, loader); - if (refresh && r.isStale(p)) { + if (refresh && r.isStale(p, this)) { cache.invalidate(p); r = cache.get(p, loader); } - } catch (ExecutionException e) { + } catch (ExecutionException | IOException e) { log.warn("Cannot load static resource " + req.getPathInfo(), e); CacheHeaders.setNotCacheable(rsp); rsp.setStatus(SC_INTERNAL_SERVER_ERROR); @@ -266,7 +270,7 @@ public abstract class ResourceServlet extends HttpServlet { public Resource call() throws IOException { try { return new Resource( - Files.getLastModifiedTime(p), + getLastModifiedTime(p), contentType(p.toString()), Files.readAllBytes(p)); } catch (NoSuchFileException e) { @@ -292,12 +296,11 @@ public abstract class ResourceServlet extends HttpServlet { this.etag = Hashing.md5().hashBytes(raw).toString(); } - boolean isStale(Path p) { - try { - return !lastModified.equals(Files.getLastModifiedTime(p)); - } catch (IOException e) { - return true; - } + boolean isStale(Path p, ResourceServlet rs) throws IOException { + FileTime t = rs.getLastModifiedTime(p); + return t.toMillis() == 0 + || lastModified.toMillis() == 0 + || !lastModified.equals(t); } } diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java index 12fe177629..e18afa07de 100644 --- a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/StaticModule.java @@ -14,18 +14,49 @@ package com.google.gerrit.httpd.raw; +import com.google.common.cache.Cache; import com.google.gerrit.httpd.raw.ResourceServlet.Resource; +import com.google.gerrit.launcher.GerritLauncher; import com.google.gerrit.server.cache.CacheModule; +import com.google.inject.Key; +import com.google.inject.Provides; +import com.google.inject.ProvisionException; +import com.google.inject.Singleton; +import com.google.inject.name.Named; +import com.google.inject.name.Names; import com.google.inject.servlet.ServletModule; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.FileSystem; import java.nio.file.Path; +import javax.servlet.http.HttpServlet; + public class StaticModule extends ServletModule { + private static final String GWT_UI_SERVLET = "GwtUiServlet"; static final String CACHE = "static_content"; + private final FileSystem warFs; + private final Path buckOut; + private final Path unpackedWar; + + public StaticModule() { + warFs = getDistributionArchive(); + if (warFs == null) { + buckOut = getDeveloperBuckOut(); + unpackedWar = makeWarTempDir(); + } else { + buckOut = null; + unpackedWar = null; + } + } + @Override protected void configureServlets() { serve("/static/*").with(SiteStaticDirectoryServlet.class); + serveGwtUi(); install(new CacheModule() { @Override protected void configure() { @@ -35,4 +66,74 @@ public class StaticModule extends ServletModule { } }); } + + private void serveGwtUi() { + serve("/gerrit_ui/*") + .with(Key.get(HttpServlet.class, Names.named(GWT_UI_SERVLET))); + if (warFs == null) { + filter("/").through(new RecompileGwtUiFilter(buckOut, unpackedWar)); + } + } + + @Provides + @Singleton + @Named(GWT_UI_SERVLET) + HttpServlet getGwtUiServlet(@Named(CACHE) Cache cache) + throws IOException { + if (warFs != null) { + return new WarGwtUiServlet(cache, warFs); + } else { + return new DeveloperGwtUiServlet(cache, unpackedWar); + } + } + + private static FileSystem getDistributionArchive() { + try { + return GerritLauncher.getDistributionArchiveFileSystem(); + } catch (IOException e) { + if ((e instanceof FileNotFoundException) + && GerritLauncher.NOT_ARCHIVED.equals(e.getMessage())) { + return null; + } else { + ProvisionException pe = + new ProvisionException("Error reading gerrit.war"); + pe.initCause(e); + throw pe; + } + } + } + + private static Path getDeveloperBuckOut() { + try { + return GerritLauncher.getDeveloperBuckOut(); + } catch (FileNotFoundException e) { + return null; + } + } + + private static Path makeWarTempDir() { + // Obtain our local temporary directory, but it comes back as a file + // so we have to switch it to be a directory post creation. + // + try { + File dstwar = GerritLauncher.createTempFile("gerrit_", "war"); + if (!dstwar.delete() || !dstwar.mkdir()) { + throw new IOException("Cannot mkdir " + dstwar.getAbsolutePath()); + } + + // Jetty normally refuses to serve out of a symlinked directory, as + // a security feature. Try to resolve out any symlinks in the path. + // + try { + return dstwar.getCanonicalFile().toPath(); + } catch (IOException e) { + return dstwar.getAbsoluteFile().toPath(); + } + } catch (IOException e) { + ProvisionException pe = + new ProvisionException("Cannot create war tempdir"); + pe.initCause(e); + throw pe; + } + } } diff --git a/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/WarGwtUiServlet.java b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/WarGwtUiServlet.java new file mode 100644 index 0000000000..45952cc41e --- /dev/null +++ b/gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/WarGwtUiServlet.java @@ -0,0 +1,47 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.httpd.raw; + +import com.google.common.cache.Cache; +import com.google.gerrit.common.TimeUtil; + +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; + +class WarGwtUiServlet extends ResourceServlet { + private static final long serialVersionUID = 1L; + + private static final FileTime NOW = FileTime.fromMillis(TimeUtil.nowMs()); + + private final FileSystem warFs; + + WarGwtUiServlet(Cache cache, FileSystem warFs) { + super(cache, false); + this.warFs = warFs; + } + + @Override + protected Path getResourcePath(String pathInfo) { + return warFs.getPath("/gerrit_ui/" + pathInfo); + } + + @Override + protected FileTime getLastModifiedTime(Path p) { + // Return initialization time of this class, since the GWT outputs from the + // build process all have mtimes of 1980/1/1. + return NOW; + } +} diff --git a/gerrit-launcher/BUCK b/gerrit-launcher/BUCK index 6281a1c0f1..687e02fd4e 100644 --- a/gerrit-launcher/BUCK +++ b/gerrit-launcher/BUCK @@ -5,6 +5,7 @@ java_library( srcs = ['src/main/java/com/google/gerrit/launcher/GerritLauncher.java'], visibility = [ '//gerrit-acceptance-tests/...', + '//gerrit-httpd:', '//gerrit-main:main_lib', '//gerrit-pgm:', ], diff --git a/gerrit-launcher/src/main/java/com/google/gerrit/launcher/GerritLauncher.java b/gerrit-launcher/src/main/java/com/google/gerrit/launcher/GerritLauncher.java index 89c8ec6cd4..fb54bcffd2 100644 --- a/gerrit-launcher/src/main/java/com/google/gerrit/launcher/GerritLauncher.java +++ b/gerrit-launcher/src/main/java/com/google/gerrit/launcher/GerritLauncher.java @@ -27,12 +27,16 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.net.JarURLConnection; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.net.URLClassLoader; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Path; import java.nio.file.Paths; import java.security.CodeSource; import java.util.ArrayList; +import java.util.Collections; import java.util.Enumeration; import java.util.List; import java.util.SortedMap; @@ -296,6 +300,7 @@ public final class GerritLauncher { } private static volatile File myArchive; + private static volatile FileSystem myArchiveFs; private static volatile File myHome; /** @@ -304,11 +309,29 @@ public final class GerritLauncher { * @return local path of the Gerrit WAR file. * @throws FileNotFoundException if the code cannot guess the location. */ - public static File getDistributionArchive() throws FileNotFoundException { - if (myArchive == null) { - myArchive = locateMyArchive(); + public static File getDistributionArchive() + throws FileNotFoundException, IOException { + File result = myArchive; + if (result == null) { + synchronized (GerritLauncher.class) { + result = myArchive; + if (result != null) { + return result; + } + result = locateMyArchive(); + myArchiveFs = FileSystems.newFileSystem( + URI.create("jar:" + result.toPath().toUri()), + Collections. emptyMap()); + myArchive = result; + } } - return myArchive; + return result; + } + + public static FileSystem getDistributionArchiveFileSystem() + throws FileNotFoundException, IOException { + getDistributionArchive(); + return myArchiveFs; } private static File locateMyArchive() throws FileNotFoundException { diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/http/jetty/JettyServer.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/http/jetty/JettyServer.java index 25b351efa6..0684650c65 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/http/jetty/JettyServer.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/http/jetty/JettyServer.java @@ -14,24 +14,15 @@ package com.google.gerrit.pgm.http.jetty; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; -import com.google.common.base.MoreObjects; import com.google.common.base.Strings; -import com.google.common.escape.Escaper; -import com.google.common.html.HtmlEscapers; -import com.google.common.io.ByteStreams; -import com.google.gerrit.common.TimeUtil; import com.google.gerrit.extensions.events.LifecycleListener; -import com.google.gerrit.launcher.GerritLauncher; import com.google.gerrit.pgm.http.jetty.HttpLog.HttpLogFactory; import com.google.gerrit.reviewdb.client.AuthType; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; -import com.google.gwtexpui.linker.server.UserAgentRule; -import com.google.gwtexpui.server.CacheHeaders; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Singleton; @@ -60,49 +51,26 @@ import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.BlockingArrayQueue; import org.eclipse.jetty.util.log.Log; -import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.eclipse.jetty.util.thread.ThreadPool; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.util.RawParseUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InterruptedIOException; -import java.io.PrintWriter; import java.lang.management.ManagementFactory; -import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.EnumSet; -import java.util.Enumeration; import java.util.HashSet; import java.util.List; -import java.util.Properties; import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import javax.servlet.DispatcherType; import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; @Singleton public class JettyServer { @@ -158,13 +126,9 @@ public class JettyServer { private boolean reverseProxy; - /** Location on disk where our WAR file was unpacked to. */ - private Resource baseResource; - @Inject JettyServer(@GerritServerConfig final Config cfg, final SitePaths site, - final JettyEnv env, final HttpLogFactory httpLogFactory) - throws MalformedURLException, IOException { + final JettyEnv env, final HttpLogFactory httpLogFactory) { this.site = site; httpd = new Server(threadPool(cfg)); @@ -372,8 +336,7 @@ public class JettyServer { return pool; } - private Handler makeContext(final JettyEnv env, final Config cfg) - throws MalformedURLException, IOException { + private Handler makeContext(final JettyEnv env, final Config cfg) { final Set paths = new HashSet<>(); for (URI u : listenURLs(cfg)) { String p = u.getPath(); @@ -408,7 +371,7 @@ public class JettyServer { } private ContextHandler makeContext(final String contextPath, - final JettyEnv env, final Config cfg) throws MalformedURLException, IOException { + final JettyEnv env, final Config cfg) { final ServletContextHandler app = new ServletContextHandler(); // This enables the use of sessions in Jetty, feature available @@ -421,12 +384,6 @@ public class JettyServer { // app.setContextPath(contextPath); - // Serve static resources directly from our JAR. This way we don't - // need to unpack them into yet another temporary directory prior to - // serving to clients. - // - app.setBaseResource(getBaseResource(app)); - // HTTP front-end filter to be used as surrogate of Apache HTTP // reverse-proxy filtering. // It is meant to be used as simpler tiny deployment of custom-made @@ -478,222 +435,4 @@ public class JettyServer { app.setWelcomeFiles(new String[0]); return app; } - - private Resource getBaseResource(ServletContextHandler app) - throws IOException { - if (baseResource == null) { - try { - baseResource = unpackWar(GerritLauncher.getDistributionArchive()); - } catch (FileNotFoundException err) { - if (GerritLauncher.NOT_ARCHIVED.equals(err.getMessage())) { - baseResource = useDeveloperBuild(app); - } else { - throw err; - } - } - } - return baseResource; - } - - private static Resource unpackWar(File srcwar) throws IOException { - File dstwar = makeWarTempDir(); - unpack(srcwar, dstwar); - return Resource.newResource(dstwar.toURI()); - } - - private static File makeWarTempDir() throws IOException { - // Obtain our local temporary directory, but it comes back as a file - // so we have to switch it to be a directory post creation. - // - File dstwar = GerritLauncher.createTempFile("gerrit_", "war"); - if (!dstwar.delete() || !dstwar.mkdir()) { - throw new IOException("Cannot mkdir " + dstwar.getAbsolutePath()); - } - - // Jetty normally refuses to serve out of a symlinked directory, as - // a security feature. Try to resolve out any symlinks in the path. - // - try { - return dstwar.getCanonicalFile(); - } catch (IOException e) { - return dstwar.getAbsoluteFile(); - } - } - - private static void unpack(File srcwar, File dstwar) throws IOException { - try (ZipFile zf = new ZipFile(srcwar)) { - final Enumeration e = zf.entries(); - while (e.hasMoreElements()) { - final ZipEntry ze = e.nextElement(); - final String name = ze.getName(); - - if (ze.isDirectory() - || name.startsWith("WEB-INF/") - || name.startsWith("META-INF/") - || name.startsWith("com/google/gerrit/launcher/") - || name.equals("Main.class")) { - continue; - } - - final File rawtmp = new File(dstwar, name); - mkdir(rawtmp.getParentFile()); - rawtmp.deleteOnExit(); - - try (FileOutputStream rawout = new FileOutputStream(rawtmp); - InputStream in = zf.getInputStream(ze)) { - final byte[] buf = new byte[4096]; - int n; - while ((n = in.read(buf, 0, buf.length)) > 0) { - rawout.write(buf, 0, n); - } - } - } - } - } - - private static void mkdir(File dir) throws IOException { - if (!dir.isDirectory()) { - mkdir(dir.getParentFile()); - if (!dir.mkdir()) { - throw new IOException("Cannot mkdir " + dir.getAbsolutePath()); - } - dir.deleteOnExit(); - } - } - - private Resource useDeveloperBuild(ServletContextHandler app) - throws IOException { - final Path dir = GerritLauncher.getDeveloperBuckOut(); - final Path gen = dir.resolve("gen"); - final Path root = dir.getParent(); - final File dstwar = makeWarTempDir(); - File ui = new File(dstwar, "gerrit_ui"); - File p = new File(ui, "permutations"); - mkdir(ui); - p.createNewFile(); - p.deleteOnExit(); - - app.addFilter(new FilterHolder(new Filter() { - private final boolean gwtuiRecompile = - System.getProperty("gerrit.disable-gwtui-recompile") == null; - private final UserAgentRule rule = new UserAgentRule(); - private final Set uaInitialized = new HashSet<>(); - private String lastTarget; - private long lastTime; - - @Override - public void doFilter(ServletRequest request, ServletResponse res, - FilterChain chain) throws IOException, ServletException { - String pkg = "gerrit-gwtui"; - String target = "ui_" + rule.select((HttpServletRequest) request); - if (gwtuiRecompile || !uaInitialized.contains(target)) { - String rule = "//" + pkg + ":" + target; - // TODO(davido): instead of assuming specific Buck's internal - // target directory for gwt_binary() artifacts, ask Buck for - // the location of user agent permutation GWT zip, e. g.: - // $ buck targets --show_output //gerrit-gwtui:ui_safari \ - // | awk '{print $2}' - String child = String.format("%s/__gwt_binary_%s__", pkg, target); - File zip = gen.resolve(child).resolve(target + ".zip").toFile(); - - synchronized (this) { - try { - build(root, gen, rule); - } catch (BuildFailureException e) { - displayFailure(rule, e.why, (HttpServletResponse) res); - return; - } - - if (!target.equals(lastTarget) || lastTime != zip.lastModified()) { - lastTarget = target; - lastTime = zip.lastModified(); - unpack(zip, dstwar); - } - } - uaInitialized.add(target); - } - chain.doFilter(request, res); - } - - private void displayFailure(String rule, byte[] why, HttpServletResponse res) - throws IOException { - res.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - res.setContentType("text/html"); - res.setCharacterEncoding(UTF_8.name()); - CacheHeaders.setNotCacheable(res); - - Escaper html = HtmlEscapers.htmlEscaper(); - try (PrintWriter w = res.getWriter()) { - w.write("BUILD FAILED"); - w.format("

%s FAILED

", html.escape(rule)); - w.write("
");
-          w.write(html.escape(RawParseUtils.decode(why)));
-          w.write("
"); - w.write(""); - } - } - - @Override - public void init(FilterConfig config) { - } - - @Override - public void destroy() { - } - }), "/", EnumSet.of(DispatcherType.REQUEST)); - return Resource.newResource(dstwar.toURI()); - } - - private static void build(Path root, Path gen, String target) - throws IOException, BuildFailureException { - log.info("buck build " + target); - Properties properties = loadBuckProperties(gen); - String buck = MoreObjects.firstNonNull(properties.getProperty("buck"), "buck"); - ProcessBuilder proc = new ProcessBuilder(buck, "build", target) - .directory(root.toFile()) - .redirectErrorStream(true); - if (properties.containsKey("PATH")) { - proc.environment().put("PATH", properties.getProperty("PATH")); - } - long start = TimeUtil.nowMs(); - Process rebuild = proc.start(); - byte[] out; - try (InputStream in = rebuild.getInputStream()) { - out = ByteStreams.toByteArray(in); - } finally { - rebuild.getOutputStream().close(); - } - - int status; - try { - status = rebuild.waitFor(); - } catch (InterruptedException e) { - throw new InterruptedIOException("interrupted waiting for " + buck); - } - if (status != 0) { - throw new BuildFailureException(out); - } - - long time = TimeUtil.nowMs() - start; - log.info(String.format("UPDATED %s in %.3fs", target, time / 1000.0)); - } - - private static Properties loadBuckProperties(Path gen) - throws FileNotFoundException, IOException { - Properties properties = new Properties(); - try (InputStream in = new FileInputStream( - gen.resolve(Paths.get("tools/buck/buck.properties")).toFile())) { - properties.load(in); - } - return properties; - } - - @SuppressWarnings("serial") - private static class BuildFailureException extends Exception { - final byte[] why; - - BuildFailureException(byte[] why) { - this.why = why; - } - } }