Merge branch 'stable-2.14' into stable-2.15

* stable-2.14: Split up the git acceptance tests Set the workspace root through a JVM property sourceRoot Document how to use RBE for the gerrit build Change-Id: Ia25eb698dea8325f1b4f0ac87fe43eb04a2e423f
2019-05-18 10:24:48 +02:00
parent ea98010756 3ca3a2e9a7
commit 1ce5c47079
9 changed files with 215 additions and 60 deletions
--- a/.bazelrc
+++ b/.bazelrc
@@ -1,6 +1,8 @@
 build --workspace_status_command=./tools/workspace-status.sh --strategy=Closure=worker
-build --disk_cache=~/.gerritcodereview/bazel-cache/cas
 build --repository_cache=~/.gerritcodereview/bazel-cache/repository
 build --experimental_strict_action_env
 build --action_env=PATH
 test --build_tests_only
+build --disk_cache=~/.gerritcodereview/bazel-cache/cas
+
+import tools/remote-bazelrc
--- a/Documentation/dev-bazel.txt
+++ b/Documentation/dev-bazel.txt
@@ -395,6 +395,43 @@ link:https://github.com/bazelbuild/bazel/issues/5139[this bazel issue] for
 details. Users should watch the cache sizes and clean them manually if
 necessary.

+
+[[RBE]]
+==== Google Remote Build Support
+
+The Bazel build can be used with Google's Remote Build Execution.
+
+
+This needs the following setup steps:
+
+```
+gcloud auth application-default login
+gcloud services enable remotebuildexecution.googleapis.com  --project=${PROJECT}
+```
+
+Create a worker pool. The instances should have at least 4 CPUs each
+for adequate performance.
+
+```
+gcloud alpha remote-build-execution worker-pools create default \
+    --project=${PROJECT} \
+    --instance=default_instance \
+    --worker-count=50 \
+    --machine-type=n1-highcpu-4 \
+    --disk-size=200
+```
+
+To use RBE, execute
+
+```
+bazel test --config=remote \
+    --remote_instance_name=projects/${PROJECT}/instances/default_instance \
+    javatests/...
+```
+
+
+
+
 GERRIT
 ------
 Part of link:index.html[Gerrit Code Review]
--- a/34
+++ b/34
@@ -5,6 +5,40 @@ load("//tools/bzl:maven_jar.bzl", "GERRIT", "MAVEN_LOCAL", "maven_jar")
 load("//lib/codemirror:cm.bzl", "CM_VERSION", "DIFF_MATCH_PATCH_VERSION")
 load("//plugins:external_plugin_deps.bzl", "external_plugin_deps")

+http_archive(
+    name = "bazel_toolchains",
+    sha256 = "88e818f9f03628eef609c8429c210ecf265ffe46c2af095f36c7ef8b1855fef5",
+    strip_prefix = "bazel-toolchains-92dd8a7a518a2fb7ba992d47c8b38299fe0be825",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz",
+        "https://github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz",
+    ],
+)
+
+load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig")
+
+# Creates a default toolchain config for RBE.
+# Use this as is if you are using the rbe_ubuntu16_04 container,
+# otherwise refer to RBE docs.
+rbe_autoconfig(name = "rbe_default")
+
+http_archive(
+    name = "bazel_toolchains",
+    sha256 = "88e818f9f03628eef609c8429c210ecf265ffe46c2af095f36c7ef8b1855fef5",
+    strip_prefix = "bazel-toolchains-92dd8a7a518a2fb7ba992d47c8b38299fe0be825",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz",
+        "https://github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz",
+    ],
+)
+
+load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig")
+
+# Creates a default toolchain config for RBE.
+# Use this as is if you are using the rbe_ubuntu16_04 container,
+# otherwise refer to RBE docs.
+rbe_autoconfig(name = "rbe_default")
+
 http_archive(
    name = "bazel_skylib",
    sha256 = "2ea8a5ed2b448baf4a6855d3ce049c4c452a6470b1efd1504fdb7c1c134d220a",
--- a/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/BUILD
+++ b/gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/git/BUILD
@@ -1,14 +1,14 @@
 load("//gerrit-acceptance-tests:tests.bzl", "acceptance_tests")

-acceptance_tests(
-    srcs = glob(["*IT.java"]),
-    group = "git",
+[acceptance_tests(
+    srcs = [f],
+    group = f[:f.index(".")],
    labels = ["git"],
    deps = [
        ":push_for_review",
        ":submodule_util",
    ],
-)
+) for f in glob(["*IT.java"])]

 java_library(
    name = "push_for_review",
--- a/gerrit-launcher/BUILD
+++ b/gerrit-launcher/BUILD
@@ -3,17 +3,5 @@
 java_library(
    name = "launcher",
    srcs = ["src/main/java/com/google/gerrit/launcher/GerritLauncher.java"],
-    resources = [":workspace-root.txt"],
-    visibility = ["//visibility:public"],
-)
-
-# The root of the workspace is non-hermetic, but we need it for
-# on-the-fly GWT recompiles and PolyGerrit updates.
-genrule(
-    name = "gen_root",
-    outs = ["workspace-root.txt"],
-    cmd = ("cat bazel-out/stable-status.txt | " +
-           "grep STABLE_WORKSPACE_ROOT | cut -d ' ' -f 2 > $@"),
-    stamp = 1,
    visibility = ["//visibility:public"],
 )
--- a/gerrit-launcher/src/main/java/com/google/gerrit/launcher/GerritLauncher.java
+++ b/gerrit-launcher/src/main/java/com/google/gerrit/launcher/GerritLauncher.java
@@ -14,7 +14,6 @@

 package com.google.gerrit.launcher;

-import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.concurrent.TimeUnit.DAYS;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;

@@ -43,7 +42,6 @@ import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Scanner;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.jar.Attributes;
@@ -624,8 +622,6 @@ public final class GerritLauncher {
    return resolveInSourceRoot("eclipse-out");
  }

-  static final String SOURCE_ROOT_RESOURCE = "/gerrit-launcher/workspace-root.txt";
-
  /**
   * Locate a path in the source tree.
   *
@@ -637,48 +633,40 @@ public final class GerritLauncher {
    // Find ourselves in the classpath, as a loose class file or jar.
    Class<GerritLauncher> self = GerritLauncher.class;

-    // If the build system provides us with a source root, use that.
-    try (InputStream stream = self.getResourceAsStream(SOURCE_ROOT_RESOURCE)) {
-      if (stream != null) {
-        try (Scanner scan = new Scanner(stream, UTF_8.name()).useDelimiter("\n")) {
-          if (scan.hasNext()) {
-            Path p = Paths.get(scan.next());
-            if (!Files.exists(p)) {
-              throw new FileNotFoundException("source root not found: " + p);
-            }
-            return p;
-          }
+    Path dir;
+    String sourceRoot = System.getProperty("sourceRoot");
+    if (sourceRoot != null) {
+      dir = Paths.get(sourceRoot);
+      if (!Files.exists(dir)) {
+        throw new FileNotFoundException("source root not found: " + dir);
+      }
+    } else {
+      URL u = self.getResource(self.getSimpleName() + ".class");
+      if (u == null) {
+        throw new FileNotFoundException("Cannot find class " + self.getName());
+      } else if ("jar".equals(u.getProtocol())) {
+        String p = u.getPath();
+        try {
+          u = new URL(p.substring(0, p.indexOf('!')));
+        } catch (MalformedURLException e) {
+          FileNotFoundException fnfe = new FileNotFoundException("Not a valid jar file: " + u);
+          fnfe.initCause(e);
+          throw fnfe;
        }
      }
-    } catch (IOException e) {
-      // not Bazel, then.
-    }
-
-    URL u = self.getResource(self.getSimpleName() + ".class");
-    if (u == null) {
-      throw new FileNotFoundException("Cannot find class " + self.getName());
-    } else if ("jar".equals(u.getProtocol())) {
-      String p = u.getPath();
-      try {
-        u = new URL(p.substring(0, p.indexOf('!')));
-      } catch (MalformedURLException e) {
-        FileNotFoundException fnfe = new FileNotFoundException("Not a valid jar file: " + u);
-        fnfe.initCause(e);
-        throw fnfe;
+      if (!"file".equals(u.getProtocol())) {
+        throw new FileNotFoundException("Cannot extract path from " + u);
      }
-    }
-    if (!"file".equals(u.getProtocol())) {
-      throw new FileNotFoundException("Cannot extract path from " + u);
-    }

-    // Pop up to the top-level source folder by looking for .buckconfig.
-    Path dir = Paths.get(u.getPath());
-    while (!Files.isRegularFile(dir.resolve("WORKSPACE"))) {
-      Path parent = dir.getParent();
-      if (parent == null) {
-        throw new FileNotFoundException("Cannot find source root from " + u);
+      // Pop up to the top-level source folder by looking for WORKSPACE.
+      dir = Paths.get(u.getPath());
+      while (!Files.isRegularFile(dir.resolve("WORKSPACE"))) {
+        Path parent = dir.getParent();
+        if (parent == null) {
+          throw new FileNotFoundException("Cannot find source root from " + u);
+        }
+        dir = parent;
      }
-      dir = parent;
    }

    Path ret = dir.resolve(name);
--- a/polygerrit-ui/README.md
+++ b/polygerrit-ui/README.md
@@ -71,7 +71,7 @@ that serves PolyGerrit:

 ```sh
 bazel build polygerrit &&
-  $(bazel info output_base)/external/local_jdk/bin/java \
+  $(bazel info output_base)/external/local_jdk/bin/java -DsourceRoot=/path/to/my/checkout \
  -jar bazel-bin/polygerrit.war daemon --polygerrit-dev \
  -d ../gerrit_testsite --console-log --show-stack-trace
 ```
--- a/tools/remote-bazelrc
+++ b/tools/remote-bazelrc
@@ -0,0 +1,107 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is auto-generated from release/bazelrc.tpl and should not be
+# modified directly.
+
+# This .bazelrc file contains all of the flags required for the provided
+# toolchain with Remote Build Execution.
+#
+# This .bazelrc file also contains all of the flags required for the local
+# docker sandboxing.
+
+# Depending on how many machines are in the remote execution instance, setting
+# this higher can make builds faster by allowing more jobs to run in parallel.
+# Setting it too high can result in jobs that timeout, however, while waiting
+# for a remote machine to execute them.
+build:remote --jobs=50
+build:remote --disk_cache=
+
+# Set several flags related to specifying the platform, toolchain and java
+# properties.
+# These flags are duplicated rather than imported from (for example)
+# %workspace%/configs/ubuntu16_04_clang/1.2/toolchain.bazelrc to make this
+# bazelrc a standalone file that can be copied more easily.
+# These flags should only be used as is for the rbe-ubuntu16-04 container
+# and need to be adapted to work with other toolchain containers.
+build:remote --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8
+build:remote --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8
+build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.2/bazel_0.25.0/default:toolchain
+build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+# Platform flags:
+# The toolchain container used for execution is defined in the target indicated
+# by "extra_execution_platforms", "host_platform" and "platforms".
+# If you are using your own toolchain container, you need to create a platform
+# target with "constraint_values" that allow for the toolchain specified with
+# "extra_toolchains" to be selected (given constraints defined in
+# "exec_compatible_with").
+# More about platforms: https://docs.bazel.build/versions/master/platforms.html
+build:remote --extra_toolchains=@bazel_toolchains//configs/ubuntu16_04_clang/1.2/bazel_0.25.0/cpp:cc-toolchain-clang-x86_64-default
+build:remote --extra_execution_platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:rbe_ubuntu1604
+build:remote --host_platform=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:rbe_ubuntu1604
+build:remote --platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:rbe_ubuntu1604
+
+# Set various strategies so that all actions execute remotely. Mixing remote
+# and local execution will lead to errors unless the toolchain and remote
+# machine exactly match the host machine.
+build:remote --spawn_strategy=remote
+build:remote --strategy=Javac=remote
+build:remote --strategy=Closure=remote
+build:remote --strategy=Genrule=remote
+build:remote --define=EXECUTOR=remote
+
+# Enable the remote cache so action results can be shared across machines,
+# developers, and workspaces.
+build:remote --remote_cache=remotebuildexecution.googleapis.com
+
+# Enable remote execution so actions are performed on the remote systems.
+build:remote --remote_executor=remotebuildexecution.googleapis.com
+
+# Enable encryption.
+build:remote --tls_enabled=true
+
+# Set a higher timeout value, just in case.
+build:remote --remote_timeout=3600
+
+# Enable authentication. This will pick up application default credentials by
+# default. You can use --auth_credentials=some_file.json to use a service
+# account credential instead.
+build:remote --auth_enabled=true
+
+# The following flags are only necessary for local docker sandboxing
+# with the rbe-ubuntu16-04 container. Use of these flags is still experimental.
+build:docker-sandbox --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8
+build:docker-sandbox --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.2:jdk8
+build:docker-sandbox --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.2/bazel_0.25.0/default:toolchain
+build:docker-sandbox --experimental_docker_image=gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:da0f21c71abce3bbb92c3a0c44c3737f007a82b60f8bd2930abc55fe64fc2729
+build:docker-sandbox --spawn_strategy=docker
+build:docker-sandbox --strategy=Javac=docker
+build:docker-sandbox --strategy=Closure=docker
+build:docker-sandbox --strategy=Genrule=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+
+# The following flags enable the remote cache so action results can be shared
+# across machines, developers, and workspaces.
+build:remote-cache --remote_cache=remotebuildexecution.googleapis.com
+build:remote-cache --tls_enabled=true
+build:remote-cache --remote_timeout=3600
+build:remote-cache --auth_enabled=true
+build:remote-cache --spawn_strategy=standalone
+build:remote-cache --strategy=Javac=standalone
+build:remote-cache --strategy=Closure=standalone
+build:remote-cache --strategy=Genrule=standalone
--- a/tools/workspace-status.sh
+++ b/tools/workspace-status.sh
@@ -19,4 +19,3 @@ for p in plugins/* ; do
  test -d "$p" || continue
  echo STABLE_BUILD_$(echo $(basename $p)_LABEL|tr '[a-z]' '[A-Z]' ) $(rev $p || echo unknown)
 done
-echo "STABLE_WORKSPACE_ROOT ${PWD}"