diff --git a/.bazelrc b/.bazelrc index 7230cf3f75..f56b683857 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,6 +1,8 @@ build --workspace_status_command=./tools/workspace-status.sh --strategy=Closure=worker -build --disk_cache=~/.gerritcodereview/bazel-cache/cas build --repository_cache=~/.gerritcodereview/bazel-cache/repository build --experimental_strict_action_env build --action_env=PATH test --build_tests_only +build --disk_cache=~/.gerritcodereview/bazel-cache/cas + +import tools/remote-bazelrc diff --git a/Documentation/dev-bazel.txt b/Documentation/dev-bazel.txt index 0a5747ceb4..39d4b7ca80 100644 --- a/Documentation/dev-bazel.txt +++ b/Documentation/dev-bazel.txt @@ -542,6 +542,43 @@ To use the binary from the Bazel build, you need to use the `run_npm_binary.py` wrapper script. For an example, see the use of `crisper` in `tools/bzl/js.bzl`. + +[[RBE]] +==== Google Remote Build Support + +The Bazel build can be used with Google's Remote Build Execution. + + +This needs the following setup steps: + +``` +gcloud auth application-default login +gcloud services enable remotebuildexecution.googleapis.com --project=${PROJECT} +``` + +Create a worker pool. The instances should have at least 4 CPUs each +for adequate performance. + +``` +gcloud alpha remote-build-execution worker-pools create default \ + --project=${PROJECT} \ + --instance=default_instance \ + --worker-count=50 \ + --machine-type=n1-highcpu-4 \ + --disk-size=200 +``` + +To use RBE, execute + +``` +bazel test --config=remote \ + --remote_instance_name=projects/${PROJECT}/instances/default_instance \ + javatests/... +``` + + + + GERRIT ------ Part of link:index.html[Gerrit Code Review] diff --git a/WORKSPACE b/WORKSPACE index 340722d09c..f2c387bd5c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -6,6 +6,40 @@ load("//tools/bzl:maven_jar.bzl", "GERRIT", "MAVEN_LOCAL", "maven_jar") load("//plugins:external_plugin_deps.bzl", "external_plugin_deps") load("//tools:nongoogle.bzl", "declare_nongoogle_deps") +http_archive( + name = "bazel_toolchains", + sha256 = "88e818f9f03628eef609c8429c210ecf265ffe46c2af095f36c7ef8b1855fef5", + strip_prefix = "bazel-toolchains-92dd8a7a518a2fb7ba992d47c8b38299fe0be825", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz", + ], +) + +load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig") + +# Creates a default toolchain config for RBE. +# Use this as is if you are using the rbe_ubuntu16_04 container, +# otherwise refer to RBE docs. +rbe_autoconfig(name = "rbe_default") + +http_archive( + name = "bazel_toolchains", + sha256 = "88e818f9f03628eef609c8429c210ecf265ffe46c2af095f36c7ef8b1855fef5", + strip_prefix = "bazel-toolchains-92dd8a7a518a2fb7ba992d47c8b38299fe0be825", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz", + ], +) + +load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig") + +# Creates a default toolchain config for RBE. +# Use this as is if you are using the rbe_ubuntu16_04 container, +# otherwise refer to RBE docs. +rbe_autoconfig(name = "rbe_default") + http_archive( name = "bazel_skylib", sha256 = "2ea8a5ed2b448baf4a6855d3ce049c4c452a6470b1efd1504fdb7c1c134d220a", @@ -1035,11 +1069,9 @@ maven_jar( sha1 = "bc8c679f6e53a51a99190a7a3108ab760b24bbf5", ) -JACKSON_VERSION = "2.9.8" - maven_jar( name = "jackson-core", - artifact = "com.fasterxml.jackson.core:jackson-core:" + JACKSON_VERSION, + artifact = "com.fasterxml.jackson.core:jackson-core:2.9.8", sha1 = "0f5a654e4675769c716e5b387830d19b501ca191", ) diff --git a/java/com/google/gerrit/launcher/BUILD b/java/com/google/gerrit/launcher/BUILD index 918a83f576..bac0c532bb 100644 --- a/java/com/google/gerrit/launcher/BUILD +++ b/java/com/google/gerrit/launcher/BUILD @@ -3,17 +3,5 @@ java_library( name = "launcher", srcs = ["GerritLauncher.java"], - resources = [":workspace-root.txt"], - visibility = ["//visibility:public"], -) - -# The root of the workspace is non-hermetic, but we need it for -# on-the-flyPolyGerrit updates. -genrule( - name = "gen_root", - outs = ["workspace-root.txt"], - cmd = ("cat bazel-out/stable-status.txt | " + - "grep STABLE_WORKSPACE_ROOT | cut -d ' ' -f 2 > $@"), - stamp = 1, visibility = ["//visibility:public"], ) diff --git a/java/com/google/gerrit/launcher/GerritLauncher.java b/java/com/google/gerrit/launcher/GerritLauncher.java index dec077a1a7..077c763a1a 100644 --- a/java/com/google/gerrit/launcher/GerritLauncher.java +++ b/java/com/google/gerrit/launcher/GerritLauncher.java @@ -46,7 +46,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.Scanner; import java.util.SortedMap; import java.util.TreeMap; import java.util.jar.Attributes; @@ -663,7 +662,6 @@ public final class GerritLauncher { } static final String SOURCE_ROOT_RESOURCE = "/com/google/gerrit/launcher/workspace-root.txt"; - /** * Locate a path in the source tree. * @@ -675,48 +673,40 @@ public final class GerritLauncher { // Find ourselves in the classpath, as a loose class file or jar. Class self = GerritLauncher.class; - // If the build system provides us with a source root, use that. - try (InputStream stream = self.getResourceAsStream(SOURCE_ROOT_RESOURCE)) { - if (stream != null) { - try (Scanner scan = new Scanner(stream, UTF_8.name()).useDelimiter("\n")) { - if (scan.hasNext()) { - Path p = Paths.get(scan.next()); - if (!Files.exists(p)) { - throw new FileNotFoundException("source root not found: " + p); - } - return p; - } + Path dir; + String sourceRoot = System.getProperty("sourceRoot"); + if (sourceRoot != null) { + dir = Paths.get(sourceRoot); + if (!Files.exists(dir)) { + throw new FileNotFoundException("source root not found: " + dir); + } + } else { + URL u = self.getResource(self.getSimpleName() + ".class"); + if (u == null) { + throw new FileNotFoundException("Cannot find class " + self.getName()); + } else if ("jar".equals(u.getProtocol())) { + String p = u.getPath(); + try { + u = new URL(p.substring(0, p.indexOf('!'))); + } catch (MalformedURLException e) { + FileNotFoundException fnfe = new FileNotFoundException("Not a valid jar file: " + u); + fnfe.initCause(e); + throw fnfe; } } - } catch (IOException e) { - // not Bazel, then. - } - - URL u = self.getResource(self.getSimpleName() + ".class"); - if (u == null) { - throw new FileNotFoundException("Cannot find class " + self.getName()); - } else if ("jar".equals(u.getProtocol())) { - String p = u.getPath(); - try { - u = new URL(p.substring(0, p.indexOf('!'))); - } catch (MalformedURLException e) { - FileNotFoundException fnfe = new FileNotFoundException("Not a valid jar file: " + u); - fnfe.initCause(e); - throw fnfe; + if (!"file".equals(u.getProtocol())) { + throw new FileNotFoundException("Cannot extract path from " + u); } - } - if (!"file".equals(u.getProtocol())) { - throw new FileNotFoundException("Cannot extract path from " + u); - } - // Pop up to the top-level source folder by looking for .buckconfig. - Path dir = Paths.get(u.getPath()); - while (!Files.isRegularFile(dir.resolve("WORKSPACE"))) { - Path parent = dir.getParent(); - if (parent == null) { - throw new FileNotFoundException("Cannot find source root from " + u); + // Pop up to the top-level source folder by looking for WORKSPACE. + dir = Paths.get(u.getPath()); + while (!Files.isRegularFile(dir.resolve("WORKSPACE"))) { + Path parent = dir.getParent(); + if (parent == null) { + throw new FileNotFoundException("Cannot find source root from " + u); + } + dir = parent; } - dir = parent; } Path ret = dir.resolve(name); diff --git a/javatests/com/google/gerrit/acceptance/git/BUILD b/javatests/com/google/gerrit/acceptance/git/BUILD index 0c43d8cd37..cabb1ba777 100644 --- a/javatests/com/google/gerrit/acceptance/git/BUILD +++ b/javatests/com/google/gerrit/acceptance/git/BUILD @@ -1,15 +1,15 @@ load("//javatests/com/google/gerrit/acceptance:tests.bzl", "acceptance_tests") -acceptance_tests( - srcs = glob(["*IT.java"]), - group = "git", +[acceptance_tests( + srcs = [f], + group = f[:f.index(".")], labels = ["git"], deps = [ ":push_for_review", ":submodule_util", "//lib/commons:lang", ], -) +) for f in glob(["*IT.java"])] java_library( name = "push_for_review", diff --git a/polygerrit-ui/README.md b/polygerrit-ui/README.md index 624451219b..cd739a1f53 100644 --- a/polygerrit-ui/README.md +++ b/polygerrit-ui/README.md @@ -72,7 +72,7 @@ that serves PolyGerrit: ```sh bazel build gerrit && - $(bazel info output_base)/external/local_jdk/bin/java \ + $(bazel info output_base)/external/local_jdk/bin/java -DsourceRoot=/path/to/my/checkout \ -jar bazel-bin/gerrit.war daemon --polygerrit-dev \ -d ../gerrit_testsite --console-log --show-stack-trace ``` diff --git a/tools/remote-bazelrc b/tools/remote-bazelrc new file mode 100644 index 0000000000..b26a8e8f6d --- /dev/null +++ b/tools/remote-bazelrc @@ -0,0 +1,107 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated from release/bazelrc.tpl and should not be +# modified directly. + +# This .bazelrc file contains all of the flags required for the provided +# toolchain with Remote Build Execution. +# +# This .bazelrc file also contains all of the flags required for the local +# docker sandboxing. + +# Depending on how many machines are in the remote execution instance, setting +# this higher can make builds faster by allowing more jobs to run in parallel. +# Setting it too high can result in jobs that timeout, however, while waiting +# for a remote machine to execute them. +build:remote --jobs=50 +build:remote --disk_cache= + +# Set several flags related to specifying the platform, toolchain and java +# properties. +# These flags are duplicated rather than imported from (for example) +# %workspace%/configs/ubuntu16_04_clang/1.2/toolchain.bazelrc to make this +# bazelrc a standalone file that can be copied more easily. +# These flags should only be used as is for the rbe-ubuntu16-04 container +# and need to be adapted to work with other toolchain containers. +build:remote --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8 +build:remote --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8 +build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 +build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 +build:remote --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.2/bazel_0.25.0/default:toolchain +build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +# Platform flags: +# The toolchain container used for execution is defined in the target indicated +# by "extra_execution_platforms", "host_platform" and "platforms". +# If you are using your own toolchain container, you need to create a platform +# target with "constraint_values" that allow for the toolchain specified with +# "extra_toolchains" to be selected (given constraints defined in +# "exec_compatible_with"). +# More about platforms: https://docs.bazel.build/versions/master/platforms.html +build:remote --extra_toolchains=@bazel_toolchains//configs/ubuntu16_04_clang/1.2/bazel_0.25.0/cpp:cc-toolchain-clang-x86_64-default +build:remote --extra_execution_platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:rbe_ubuntu1604 +build:remote --host_platform=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:rbe_ubuntu1604 +build:remote --platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:rbe_ubuntu1604 + +# Set various strategies so that all actions execute remotely. Mixing remote +# and local execution will lead to errors unless the toolchain and remote +# machine exactly match the host machine. +build:remote --spawn_strategy=remote +build:remote --strategy=Javac=remote +build:remote --strategy=Closure=remote +build:remote --strategy=Genrule=remote +build:remote --define=EXECUTOR=remote + +# Enable the remote cache so action results can be shared across machines, +# developers, and workspaces. +build:remote --remote_cache=remotebuildexecution.googleapis.com + +# Enable remote execution so actions are performed on the remote systems. +build:remote --remote_executor=remotebuildexecution.googleapis.com + +# Enable encryption. +build:remote --tls_enabled=true + +# Set a higher timeout value, just in case. +build:remote --remote_timeout=3600 + +# Enable authentication. This will pick up application default credentials by +# default. You can use --auth_credentials=some_file.json to use a service +# account credential instead. +build:remote --auth_enabled=true + +# The following flags are only necessary for local docker sandboxing +# with the rbe-ubuntu16-04 container. Use of these flags is still experimental. +build:docker-sandbox --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8 +build:docker-sandbox --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8 +build:docker-sandbox --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.2/bazel_0.25.0/default:toolchain +build:docker-sandbox --experimental_docker_image=gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:da0f21c71abce3bbb92c3a0c44c3737f007a82b60f8bd2930abc55fe64fc2729 +build:docker-sandbox --spawn_strategy=docker +build:docker-sandbox --strategy=Javac=docker +build:docker-sandbox --strategy=Closure=docker +build:docker-sandbox --strategy=Genrule=docker +build:docker-sandbox --define=EXECUTOR=remote +build:docker-sandbox --experimental_docker_verbose +build:docker-sandbox --experimental_enable_docker_sandbox + +# The following flags enable the remote cache so action results can be shared +# across machines, developers, and workspaces. +build:remote-cache --remote_cache=remotebuildexecution.googleapis.com +build:remote-cache --tls_enabled=true +build:remote-cache --remote_timeout=3600 +build:remote-cache --auth_enabled=true +build:remote-cache --spawn_strategy=standalone +build:remote-cache --strategy=Javac=standalone +build:remote-cache --strategy=Closure=standalone +build:remote-cache --strategy=Genrule=standalone diff --git a/tools/workspace-status.sh b/tools/workspace-status.sh index af6e180ca3..2b1a4ba2f5 100755 --- a/tools/workspace-status.sh +++ b/tools/workspace-status.sh @@ -19,4 +19,3 @@ for p in plugins/* ; do test -d "$p" || continue echo STABLE_BUILD_$(echo $(basename $p)_LABEL|tr '[a-z]' '[A-Z]' ) $(rev $p || echo unknown) done -echo "STABLE_WORKSPACE_ROOT ${PWD}"