bazel: abstract the build system in static serving.

Provide BazelBuild to stub out the Bazel build.

Tested:

Production use case:
1.) bazel build polygerrit && \
    $(bazel info output_base)/external/local_jdk/bin/java \
     -jar bazel-bin/polygerrit.war daemon -d ../test_site \
     --console-log --show-stack-trace

Development mode use cases:
2.) bazel build polygerrit \
      //polygerrit-ui:polygerrit_components.bower_components.zip &&
    $(bazel info output_base)/external/local_jdk/bin/java \
     -jar bazel-bin/polygerrit.war daemon \
     --polygerrit-dev -d ../gerrit_testsite --console-log --show-stack-trace

checked that updates under polygerrit-ui/app/index.html are served
live.

3.) Run tools/eclipse/project.py, started gwt_daemon launcher,
verified that it worked.

4.) Run tools/eclipse/project.py, started gerit_gwt_debug launcher,
verified that GWT SDM worked.

Change-Id: I9d105e00e953b63c78306e9e37d5152673627727
This commit is contained in:
Han-Wen Nienhuys 2016-10-27 15:43:10 +02:00 committed by David Ostrovsky
parent d2b6d71a1e
commit e6c7e629c6
12 changed files with 343 additions and 118 deletions

View File

@ -0,0 +1,57 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.httpd.raw;
import java.io.IOException;
import java.nio.file.Path;
public class BazelBuild implements BuildSystem {
private final Path sourceRoot;
public BazelBuild(Path sourceRoot) {
this.sourceRoot = sourceRoot;
}
@Override
public void build(Label l) throws IOException, BuildFailureException {
throw new BuildFailureException("not implemented yet.".getBytes());
}
@Override
public String buildCommand(Label l) {
return "bazel build " + l.toString();
}
@Override
public Path targetPath(Label l) {
return sourceRoot.resolve("bazel-bin").resolve(l.pkg).resolve(l.name);
}
@Override
public Label gwtZipLabel(String agent) {
return new Label("gerrit-gwtui", "ui_" + agent + ".zip");
}
@Override
public Label polygerritComponents() {
return new Label("polygerrit-ui",
"polygerrit_components.bower_components.zip");
}
@Override
public Label fontZipLabel() {
return new Label("polygerrit-ui", "fonts.zip");
}
}

View File

@ -15,9 +15,11 @@
package com.google.gerrit.httpd.raw;
import com.google.common.cache.Cache;
import com.google.gerrit.httpd.raw.BuildSystem.Label;
import com.google.gerrit.launcher.GerritLauncher;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
@ -26,24 +28,33 @@ class BowerComponentsDevServlet extends ResourceServlet {
private static final long serialVersionUID = 1L;
private final Path bowerComponents;
private final String buildCommand;
private final Path zip;
BowerComponentsDevServlet(Cache<Path, Resource> cache, Path buckOut)
throws IOException {
BowerComponentsDevServlet(Cache<Path, Resource> cache,
BuildSystem builder) throws IOException {
super(cache, true);
Objects.requireNonNull(buckOut);
Path zip = buckOut.resolve("gen")
.resolve("polygerrit-ui")
.resolve("polygerrit_components")
.resolve("polygerrit_components.bower_components.zip");
Objects.requireNonNull(builder);
Label pgLabel = builder.polygerritComponents();
buildCommand = builder.buildCommand(pgLabel);
bowerComponents = GerritLauncher
.newZipFileSystem(zip)
.getPath("/");
zip = builder.targetPath(pgLabel);
if (zip == null || !Files.exists(zip)) {
bowerComponents = null;
} else {
bowerComponents = GerritLauncher
.newZipFileSystem(zip)
.getPath("/");
}
}
@Override
protected Path getResourcePath(String pathInfo) throws IOException {
if (bowerComponents == null) {
throw new IOException("No polymer components found: " + zip
+ ". Run `" + buildCommand + "`?");
}
return bowerComponents.resolve(pathInfo);
}
}

View File

@ -15,41 +15,39 @@
package com.google.gerrit.httpd.raw;
import static com.google.common.base.MoreObjects.firstNonNull;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.escape.Escaper;
import com.google.common.html.HtmlEscapers;
import com.google.common.io.ByteStreams;
import com.google.gerrit.common.TimeUtil;
import com.google.gwtexpui.server.CacheHeaders;
import org.eclipse.jgit.util.RawParseUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import javax.servlet.http.HttpServletResponse;
class BuckUtils {
class BuckUtils implements BuildSystem {
private static final Logger log =
LoggerFactory.getLogger(BuckUtils.class);
private final Path sourceRoot;
static void build(Path root, Path gen, String target)
BuckUtils(Path sourceRoot) {
this.sourceRoot = sourceRoot;
}
@Override
public void build(Label label)
throws IOException, BuildFailureException {
log.info("buck build " + target);
Properties properties = loadBuckProperties(gen);
log.info("buck build " + label.fullName());
Properties properties = loadBuckProperties(
sourceRoot.resolve("buck-out/gen/tools/buck/buck.properties"));
String buck = firstNonNull(properties.getProperty("buck"), "buck");
ProcessBuilder proc = new ProcessBuilder(buck, "build", target)
.directory(root.toFile())
ProcessBuilder proc = new ProcessBuilder(buck, "build", label.fullName())
.directory(sourceRoot.toFile())
.redirectErrorStream(true);
if (properties.containsKey("PATH")) {
proc.environment().put("PATH", properties.getProperty("PATH"));
@ -74,13 +72,14 @@ class BuckUtils {
}
long time = TimeUtil.nowMs() - start;
log.info(String.format("UPDATED %s in %.3fs", target, time / 1000.0));
log.info(String.format("UPDATED %s in %.3fs", label.fullName(),
time / 1000.0));
}
private static Properties loadBuckProperties(Path gen) throws IOException {
private static Properties loadBuckProperties(Path propPath)
throws IOException {
Properties properties = new Properties();
Path p = gen.resolve(Paths.get("tools/buck/buck.properties"));
try (InputStream in = Files.newInputStream(p)) {
try (InputStream in = Files.newInputStream(propPath)) {
properties.load(in);
} catch (NoSuchFileException e) {
// Ignore; will be run from PATH, with a descriptive error if it fails.
@ -88,31 +87,38 @@ class BuckUtils {
return properties;
}
static void displayFailure(String rule, byte[] why, HttpServletResponse res)
throws IOException {
res.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
res.setContentType("text/html");
res.setCharacterEncoding(UTF_8.name());
CacheHeaders.setNotCacheable(res);
Escaper html = HtmlEscapers.htmlEscaper();
try (PrintWriter w = res.getWriter()) {
w.write("<html><title>BUILD FAILED</title><body>");
w.format("<h1>%s FAILED</h1>", html.escape(rule));
w.write("<pre>");
w.write(html.escape(RawParseUtils.decode(why)));
w.write("</pre>");
w.write("</body></html>");
}
@Override
public Path targetPath(Label label) {
return sourceRoot.resolve("buck-out")
.resolve("gen").resolve(label.artifact);
}
static class BuildFailureException extends Exception {
private static final long serialVersionUID = 1L;
@Override
public String buildCommand(Label l) {
return "buck build " + l.toString();
}
final byte[] why;
@Override
public Label gwtZipLabel(String agent) {
// TODO(davido): instead of assuming specific Buck's internal
// target directory for gwt_binary() artifacts, ask Buck for
// the location of user agent permutation GWT zip, e. g.:
// $ buck targets --show_output //gerrit-gwtui:ui_safari \
// | awk '{print $2}'
String t = "ui_" + agent;
return new BuildSystem.Label("gerrit-gwtui", t,
String.format("gerrit-gwtui/__gwt_binary_%s__/%s.zip", t, t));
}
BuildFailureException(byte[] why) {
this.why = why;
}
@Override
public Label polygerritComponents() {
return new Label("polygerrit-ui", "polygerrit_components",
"polygerrit-ui/polygerrit_components/" +
"polygerrit_components.bower_components.zip");
}
@Override
public Label fontZipLabel() {
return new Label("polygerrit-ui", "fonts", "polygerrit-ui/fonts/fonts.zip");
}
}

View File

@ -0,0 +1,114 @@
// Copyright (C) 2016 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.httpd.raw;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.escape.Escaper;
import com.google.common.html.HtmlEscapers;
import com.google.gwtexpui.server.CacheHeaders;
import org.eclipse.jgit.util.RawParseUtils;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Path;
import javax.servlet.http.HttpServletResponse;
public interface BuildSystem {
// Represents a label in either buck or bazel.
class Label {
protected final String pkg;
protected final String name;
// Regrettably, buck confounds rule names and artifact names,
// and so we have to lug this along. Non-null only for Buck; in that case,
// holds the path relative to buck-out/gen/
protected final String artifact;
public String fullName() {
return "//" + pkg + ":" + name;
}
@Override
public String toString() {
String s = fullName();
if (!name.equals(artifact)) {
s += "(" + artifact + ")";
}
return s;
}
// Label in Buck style.
Label(String pkg, String name, String artifact) {
this.name = name;
this.pkg = pkg;
this.artifact = artifact;
}
// Label in Bazel style.
Label(String pkg, String name) {
this(pkg, name, name);
}
}
class BuildFailureException extends Exception {
private static final long serialVersionUID = 1L;
final byte[] why;
BuildFailureException(byte[] why) {
this.why = why;
}
public void display(String rule, HttpServletResponse res)
throws IOException {
res.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
res.setContentType("text/html");
res.setCharacterEncoding(UTF_8.name());
CacheHeaders.setNotCacheable(res);
Escaper html = HtmlEscapers.htmlEscaper();
try (PrintWriter w = res.getWriter()) {
w.write("<html><title>BUILD FAILED</title><body>");
w.format("<h1>%s FAILED</h1>", html.escape(rule));
w.write("<pre>");
w.write(html.escape(RawParseUtils.decode(why)));
w.write("</pre>");
w.write("</body></html>");
}
}
}
/** returns the command to build given target */
String buildCommand(Label l);
/** builds the given label. */
void build(Label l) throws IOException, BuildFailureException;
/** returns the root relative path to the artifact for the given label */
Path targetPath(Label l);
/** Label for the agent specific GWT zip. */
Label gwtZipLabel(String agent);
/** Label for the polygerrit component zip. */
Label polygerritComponents();
/** Label for the fonts zip file. */
Label fontZipLabel();
}

View File

@ -27,18 +27,15 @@ class FontsDevServlet extends ResourceServlet {
private final Path fonts;
FontsDevServlet(Cache<Path, Resource> cache, Path buckOut)
FontsDevServlet(Cache<Path, Resource> cache, BuildSystem builder)
throws IOException {
super(cache, true);
Objects.requireNonNull(buckOut);
Objects.requireNonNull(builder);
Path zip = buckOut.resolve("gen")
.resolve("polygerrit-ui")
.resolve("fonts")
.resolve("fonts.zip");
fonts = GerritLauncher
.newZipFileSystem(zip)
.getPath("/");
Path zip = builder.targetPath(builder.fontZipLabel());
Objects.requireNonNull(zip);
fonts = GerritLauncher.newZipFileSystem(zip).getPath("/");
}
@Override

View File

@ -14,7 +14,7 @@
package com.google.gerrit.httpd.raw;
import com.google.gerrit.httpd.raw.BuckUtils.BuildFailureException;
import com.google.gerrit.httpd.raw.BuildSystem.Label;
import com.google.gwtexpui.linker.server.UserAgentRule;
import java.io.File;
@ -43,48 +43,40 @@ class RecompileGwtUiFilter implements Filter {
private final UserAgentRule rule = new UserAgentRule();
private final Set<String> uaInitialized = new HashSet<>();
private final Path unpackedWar;
private final Path gen;
private final Path root;
private final BuildSystem builder;
private String lastTarget;
private String lastAgent;
private long lastTime;
RecompileGwtUiFilter(Path buckOut, Path unpackedWar) {
RecompileGwtUiFilter(BuildSystem builder, Path unpackedWar) {
this.builder = builder;
this.unpackedWar = unpackedWar;
gen = buckOut.resolve("gen");
root = buckOut.getParent();
}
@Override
public void doFilter(ServletRequest request, ServletResponse res,
FilterChain chain) throws IOException, ServletException {
String pkg = "gerrit-gwtui";
String target = "ui_" + rule.select((HttpServletRequest) request);
if (gwtuiRecompile || !uaInitialized.contains(target)) {
String rule = "//" + pkg + ":" + target;
// TODO(davido): instead of assuming specific Buck's internal
// target directory for gwt_binary() artifacts, ask Buck for
// the location of user agent permutation GWT zip, e. g.:
// $ buck targets --show_output //gerrit-gwtui:ui_safari \
// | awk '{print $2}'
String child = String.format("%s/__gwt_binary_%s__", pkg, target);
File zip = gen.resolve(child).resolve(target + ".zip").toFile();
String agent = rule.select((HttpServletRequest) request);
if (unpackedWar != null
&& (gwtuiRecompile || !uaInitialized.contains(agent))) {
Label label = builder.gwtZipLabel(agent);
File zip = builder.targetPath(label).toFile();
synchronized (this) {
try {
BuckUtils.build(root, gen, rule);
} catch (BuildFailureException e) {
BuckUtils.displayFailure(rule, e.why, (HttpServletResponse) res);
builder.build(label);
} catch (BuildSystem.BuildFailureException e) {
e.display(label.toString(), (HttpServletResponse) res);
return;
}
if (!target.equals(lastTarget) || lastTime != zip.lastModified()) {
lastTarget = target;
if (!agent.equals(lastAgent) || lastTime != zip.lastModified()) {
lastAgent = agent;
lastTime = zip.lastModified();
unpack(zip, unpackedWar.toFile());
}
}
uaInitialized.add(target);
uaInitialized.add(agent);
}
chain.doFilter(request, res);
}

View File

@ -221,8 +221,7 @@ public class StaticModule extends ServletModule {
if (p.unpackedWar != null) {
return p.unpackedWar.resolve(name);
}
return p.buckOut.resolveSibling("gerrit-war").resolve("src")
.resolve("main").resolve("webapp").resolve(name);
return p.sourceRoot.resolve("gerrit-war/src/main/webapp/" + name);
}
}
@ -233,7 +232,7 @@ public class StaticModule extends ServletModule {
.with(Key.get(HttpServlet.class, Names.named(GWT_UI_SERVLET)));
Paths p = getPaths();
if (p.isDev()) {
filter("/").through(new RecompileGwtUiFilter(p.buckOut, p.unpackedWar));
filter("/").through(new RecompileGwtUiFilter(p.builder, p.unpackedWar));
}
}
@ -286,7 +285,7 @@ public class StaticModule extends ServletModule {
BowerComponentsDevServlet getBowerComponentsServlet(
@Named(CACHE) Cache<Path, Resource> cache) throws IOException {
return getPaths().isDev()
? new BowerComponentsDevServlet(cache, getPaths().buckOut)
? new BowerComponentsDevServlet(cache, getPaths().builder)
: null;
}
@ -295,19 +294,19 @@ public class StaticModule extends ServletModule {
FontsDevServlet getFontsServlet(
@Named(CACHE) Cache<Path, Resource> cache) throws IOException {
return getPaths().isDev()
? new FontsDevServlet(cache, getPaths().buckOut)
? new FontsDevServlet(cache, getPaths().builder)
: null;
}
private Path polyGerritBasePath() {
Paths p = getPaths();
if (options.forcePolyGerritDev()) {
checkArgument(p.buckOut != null,
"no buck-out directory found for PolyGerrit developer mode");
checkArgument(p.sourceRoot != null,
"no source root directory found for PolyGerrit developer mode");
}
if (p.isDev()) {
return p.buckOut.getParent().resolve("polygerrit-ui").resolve("app");
return p.sourceRoot.resolve("polygerrit-ui").resolve("app");
}
return p.warFs != null
@ -318,7 +317,8 @@ public class StaticModule extends ServletModule {
private static class Paths {
private final FileSystem warFs;
private final Path buckOut;
private final BuildSystem builder;
private final Path sourceRoot;
private final Path unpackedWar;
private final boolean development;
@ -338,28 +338,42 @@ public class StaticModule extends ServletModule {
.getParentFile()
.getParentFile()
.toURI());
buckOut = null;
sourceRoot = null;
development = false;
builder = null;
return;
}
warFs = getDistributionArchive(launcherLoadedFrom);
if (warFs == null) {
buckOut = getDeveloperBuckOut();
unpackedWar = makeWarTempDir();
development = true;
} else if (options.forcePolyGerritDev()) {
buckOut = getDeveloperBuckOut();
unpackedWar = null;
development = true;
} else {
buckOut = null;
unpackedWar = null;
development = false;
sourceRoot = null;
builder = null;
return;
}
} catch (IOException e) {
throw new ProvisionException(
"Error initializing static content paths", e);
}
sourceRoot = getSourseRootOrNull();
builder = GerritLauncher.isBazel()
? new BazelBuild(sourceRoot)
: new BuckUtils(sourceRoot);
}
private static Path getSourseRootOrNull() {
try {
return GerritLauncher.resolveInSourceRoot(".");
} catch (FileNotFoundException e) {
return null;
}
}
private FileSystem getDistributionArchive(File war) throws IOException {
@ -390,14 +404,6 @@ public class StaticModule extends ServletModule {
return development;
}
private Path getDeveloperBuckOut() {
try {
return GerritLauncher.getDeveloperBuckOut();
} catch (FileNotFoundException e) {
return null;
}
}
private Path makeWarTempDir() {
// Obtain our local temporary directory, but it comes back as a file
// so we have to switch it to be a directory post creation.

View File

@ -3,5 +3,17 @@
java_library(
name = 'launcher',
srcs = ['src/main/java/com/google/gerrit/launcher/GerritLauncher.java'],
resources = [':workspace-root.txt'],
visibility = ['//visibility:public'],
)
# The root of the workspace is non-hermetic, but we need it for
# on-the-fly GWT recompiles and PolyGerrit updates.
genrule(
name = 'gen_root',
stamp = 1,
cmd = ("cat bazel-out/stable-status.txt | " +
"grep STABLE_WORKSPACE_ROOT | cut -d ' ' -f 2 > $@"),
outs = ['workspace-root.txt'],
visibility = ['//visibility:public'],
)

View File

@ -16,6 +16,7 @@ package com.google.gerrit.launcher;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.File;
import java.io.FileNotFoundException;
@ -42,6 +43,7 @@ import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.jar.Attributes;
@ -622,20 +624,47 @@ public final class GerritLauncher {
return resolveInSourceRoot("eclipse-out");
}
/**
* Locate the path of the {@code buck-out} directory in a source tree.
*
* @return local path of the {@code buck-out} directory in a source tree.
* @throws FileNotFoundException if the directory cannot be found.
*/
public static Path getDeveloperBuckOut() throws FileNotFoundException {
return resolveInSourceRoot("buck-out");
static String SOURCE_ROOT_RESOURCE = "/gerrit-launcher/workspace-root.txt";
/** returns whether we're running out of a bazel build. */
public static boolean isBazel() {
Class<GerritLauncher> self = GerritLauncher.class;
URL rootURL = self.getResource(SOURCE_ROOT_RESOURCE);
return rootURL != null;
}
private static Path resolveInSourceRoot(String name)
/**
* Locate a path in the source tree.
*
* @return local path of the {@code name} directory in a source tree.
* @throws FileNotFoundException if the directory cannot be found.
*/
public static Path resolveInSourceRoot(String name)
throws FileNotFoundException {
// Find ourselves in the classpath, as a loose class file or jar.
Class<GerritLauncher> self = GerritLauncher.class;
// If the build system provides us with a source root, use that.
try (InputStream stream = self.getResourceAsStream(SOURCE_ROOT_RESOURCE)) {
System.err.println("URL: " + stream);
if (stream != null) {
try (Scanner scan =
new Scanner(stream, UTF_8.name()).useDelimiter("\n")) {
if (scan.hasNext()) {
Path p = Paths.get(scan.next());
if (!Files.exists(p)) {
throw new FileNotFoundException(
"source root not found: " + p);
}
return p;
}
}
}
} catch (IOException e) {
// not Bazel, then.
}
URL u = self.getResource(self.getSimpleName() + ".class");
if (u == null) {
throw new FileNotFoundException("Cannot find class " + self.getName());
@ -674,7 +703,7 @@ public final class GerritLauncher {
private static ClassLoader useDevClasspath()
throws MalformedURLException, FileNotFoundException {
Path out = getDeveloperEclipseOut();
Path out = resolveInSourceRoot("eclipse-out");
List<URL> dirs = new ArrayList<>();
dirs.add(out.resolve("classes").toUri().toURL());
ClassLoader cl = GerritLauncher.class.getClassLoader();

View File

@ -6,7 +6,7 @@ load("//tools/bzl:js.bzl", "bower_component_bundle")
load('//tools/bzl:genrule2.bzl', 'genrule2')
bower_component_bundle(
name = "polygerrit_components",
name = "polygerrit_components.bower_components",
deps = [
'//lib/js:es6-promise',
'//lib/js:fetch',

View File

@ -18,7 +18,7 @@ vulcanize(
'test/**',
'**/*_test.html',
]),
deps = [ "//polygerrit-ui:polygerrit_components"],
deps = [ "//polygerrit-ui:polygerrit_components.bower_components"],
)
filegroup(
@ -65,7 +65,7 @@ bower_component_bundle(
name = 'test_components',
testonly = 1,
deps = [
'//polygerrit-ui:polygerrit_components',
'//polygerrit-ui:polygerrit_components.bower_components',
'//lib/js:iron-test-helpers',
'//lib/js:test-fixture',
'//lib/js:web-component-tester',

View File

@ -19,3 +19,4 @@ for p in plugins/* ; do
test -d "$p" || continue
echo STABLE_BUILD_$(echo $(basename $p)_LABEL|tr [a-z] [A-Z]) $(rev $p)
done
echo "STABLE_WORKSPACE_ROOT ${PWD}"