Bazel: Generate Eclipse classpath

To guess what build system is used, we create now .primary_build_tool
file in the root of the project during the eclipse classpath generation.

Change the working directory for GWT SDM session to be .gwt_work_dir.
The reason for that Bazel doesn't allow to write to bazel-out.

Change-Id: I984068350244ee9d66807e4bc8c6779b34a26bab

.gitignore

@@ -32,3 +32,5 @@
 *.asc
 /bin/
 *~
+.primary_build_tool
+.gwt_work_dir

Documentation/dev-bazel.txt

@@ -3,7 +3,6 @@
 Bazel build is experimental. Major missing parts:
 
 * Custom plugins
-* Eclipse project generation.
 * Test suites for SSH, acceptance, etc.
 * tag tests as slow, flaky, etc.
 
@@ -157,6 +156,25 @@ plugin](https://ij.bazel.io). Do the following:
   * Select "Workspace": (directory holding gerrit source)
   * Select "project view: generate from BUILD": (enter top level BUILD file)
 
+
+=== Eclipse
+
+==== Generating the Eclipse Project
+
+Create the Eclipse project:
+
+----
+  tools/eclipse/project_bzl.py
+----
+
+and then follow the link:dev-eclipse.html#setup[setup instructions].
+
+==== Refreshing the Classpath
+
+If an updated classpath is needed, the Eclipse project can be
+refreshed and missing dependency JARs can be downloaded by running
+`project_bzl.py` again.
+
 [[documentation]]
 === Documentation
 

gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/BazelBuild.java

@@ -14,8 +14,11 @@
 
 package com.google.gerrit.httpd.raw;
 
+import static com.google.common.base.MoreObjects.firstNonNull;
+
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Properties;
 
 public class BazelBuild extends BuildSystem {
   public BazelBuild(Path sourceRoot) {
@@ -24,7 +27,13 @@ public class BazelBuild extends BuildSystem {
 
   @Override
   protected ProcessBuilder newBuildProcess(Label label) throws IOException {
-    ProcessBuilder proc = new ProcessBuilder("bazel", "build", label.fullName());
+    Properties properties = loadBuildProperties(
+        sourceRoot.resolve(".primary_build_tool"));
+    String buck = firstNonNull(properties.getProperty("bazel"), "bazel");
+    ProcessBuilder proc = new ProcessBuilder(buck, "build", label.fullName());
+    if (properties.containsKey("PATH")) {
+      proc.environment().put("PATH", properties.getProperty("PATH"));
+    }
     return proc;
   }
 

gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/BuckUtils.java

@@ -17,9 +17,6 @@ package com.google.gerrit.httpd.raw;
 import static com.google.common.base.MoreObjects.firstNonNull;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.util.Properties;
 
@@ -30,7 +27,7 @@ class BuckUtils extends BuildSystem {
 
   @Override
   protected ProcessBuilder newBuildProcess(Label label) throws IOException {
-    Properties properties = loadBuckProperties(
+    Properties properties = loadBuildProperties(
         sourceRoot.resolve("buck-out/gen/tools/buck/buck.properties"));
     String buck = firstNonNull(properties.getProperty("buck"), "buck");
     ProcessBuilder proc = new ProcessBuilder(buck, "build", label.fullName());
@@ -40,17 +37,6 @@ class BuckUtils extends BuildSystem {
     return proc;
   }
 
-  private static Properties loadBuckProperties(Path propPath)
-      throws IOException {
-    Properties properties = new Properties();
-    try (InputStream in = Files.newInputStream(propPath)) {
-      properties.load(in);
-    } catch (NoSuchFileException e) {
-      // Ignore; will be run from PATH, with a descriptive error if it fails.
-    }
-    return properties;
-  }
-
   @Override
   public Path targetPath(Label label) {
     return sourceRoot.resolve("buck-out")

gerrit-httpd/src/main/java/com/google/gerrit/httpd/raw/BuildSystem.java

@@ -30,7 +30,10 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InterruptedIOException;
 import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.util.Properties;
 
 import javax.servlet.http.HttpServletResponse;
 
@@ -46,6 +49,17 @@ public abstract class BuildSystem {
 
   protected abstract ProcessBuilder newBuildProcess(Label l) throws IOException;
 
+  protected static Properties loadBuildProperties(Path propPath)
+      throws IOException {
+    Properties properties = new Properties();
+    try (InputStream in = Files.newInputStream(propPath)) {
+      properties.load(in);
+    } catch (NoSuchFileException e) {
+      // Ignore; will be run from PATH, with a descriptive error if it fails.
+    }
+    return properties;
+  }
+
   // builds the given label.
   public void build(Label label)
       throws IOException, BuildFailureException {

gerrit-launcher/src/main/java/com/google/gerrit/launcher/GerritLauncher.java

@@ -625,12 +625,32 @@ public final class GerritLauncher {
   }
 
   static String SOURCE_ROOT_RESOURCE = "/gerrit-launcher/workspace-root.txt";
+  static String PRIMARY_BUILD_TOOL = ".primary_build_tool";
 
   /** returns whether we're running out of a bazel build. */
   public static boolean isBazel() {
     Class<GerritLauncher> self = GerritLauncher.class;
     URL rootURL = self.getResource(SOURCE_ROOT_RESOURCE);
-    return rootURL != null;
+    if (rootURL != null) {
+      return true;
+    }
+
+    Path p = null;
+    try {
+      p = resolveInSourceRoot("eclipse-out");
+      Path path = p.getParent().resolve(PRIMARY_BUILD_TOOL);
+      if (Files.exists(path)) {
+        String content = new String(Files.readAllBytes(path));
+        if (content.toLowerCase().contains("bazel")) {
+          return true;
+        }
+      }
+    } catch (IOException e) {
+      // Ignore
+    }
+
+    // Not Bazel then
+    return false;
   }
 
   /**
@@ -685,7 +705,8 @@ public final class GerritLauncher {
 
     // Pop up to the top-level source folder by looking for .buckconfig.
     Path dir = Paths.get(u.getPath());
-    while (!Files.isRegularFile(dir.resolve(".buckconfig"))) {
+    while (!Files.isRegularFile(dir.resolve(".buckconfig"))
+        && !Files.isRegularFile(dir.resolve("WORKSPACE"))) {
       Path parent = dir.getParent();
       if (parent == null) {
         throw new FileNotFoundException("Cannot find source root from " + u);

tools/bzl/classpath.bzl

@@ -2,13 +2,16 @@
 def _classpath_collector(ctx):
     all = set()
     for d in ctx.attr.deps:
-        all += d.java.compilation_info.runtime_classpath
+        if hasattr(d, 'java'):
+            all += d.java.transitive_runtime_deps
+            all += d.java.compilation_info.runtime_classpath
+        elif hasattr(d, 'files'):
+            all += d.files
 
     as_strs = [c.path for c in all]
     ctx.file_action(output= ctx.outputs.runtime,
                     content="\n".join(sorted(as_strs)))
 
-
 classpath_collector = rule(
     implementation = _classpath_collector,
     attrs = {

tools/eclipse/BUILD

@@ -0,0 +1,67 @@
+load('//tools/bzl:pkg_war.bzl', 'LIBS', 'PGMLIBS')
+load('//tools/bzl:classpath.bzl', 'classpath_collector')
+
+PROVIDED_DEPS = [
+  '//lib/bouncycastle:bcprov',
+  '//lib/bouncycastle:bcpg',
+  '//lib/bouncycastle:bcpkix',
+]
+
+TEST_DEPS = [
+  '//gerrit-gpg:gpg_tests',
+  '//gerrit-gwtui:ui_tests',
+  '//gerrit-httpd:httpd_tests',
+  '//gerrit-patch-jgit:jgit_patch_tests',
+  '//gerrit-reviewdb:client_tests',
+  '//gerrit-server:server_tests',
+]
+
+DEPS = [
+  '//gerrit-acceptance-tests:lib',
+  '//gerrit-gwtdebug:gwtdebug',
+  '//gerrit-gwtui:ui_module',
+  '//gerrit-main:main_lib',
+  '//gerrit-plugin-gwtui:gwtui-api-lib',
+  '//gerrit-server:server',
+  # TODO(davido): figure out why it's not reached through test dependencies
+  '//lib:jimfs',
+  '//lib/asciidoctor:asciidoc_lib',
+  '//lib/asciidoctor:doc_indexer_lib',
+  '//lib/auto:auto-value',
+  '//lib/gwt:ant',
+  '//lib/gwt:colt',
+  '//lib/gwt:javax-validation',
+  '//lib/gwt:javax-validation_src',
+  '//lib/gwt:jsinterop-annotations',
+  '//lib/gwt:jsinterop-annotations_src',
+  '//lib/gwt:tapestry',
+  '//lib/gwt:w3c-css-sac',
+  '//lib/jetty:servlets',
+  '//lib/prolog:compiler_lib',
+  # TODO(davido): I do not understand why it must be on the Eclipse classpath
+  #'//Documentation:index',
+]
+
+java_library(
+  name = 'classpath',
+  runtime_deps = LIBS + PGMLIBS + DEPS,
+  testonly = 1,
+)
+
+classpath_collector(
+  name = 'main_classpath_collect',
+  deps = LIBS + PGMLIBS + DEPS + PROVIDED_DEPS,
+  testonly = 1,
+  # TODO(davido): Handle plugins
+  #scan_plugins(),
+)
+
+classpath_collector(
+  name = "gwt_classpath_collect",
+  deps = ["//gerrit-gwtui:ui_module"],
+)
+
+classpath_collector(
+  name = "autovalue_classpath_collect",
+  deps = ["//lib/auto:auto-value"],
+)

tools/eclipse/gerrit_daemon.launch

@@ -13,5 +13,5 @@
 <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="Main"/>
 <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="daemon --console-log --show-stack-trace -d ${resource_loc:/gerrit}/../gerrit_testsite"/>
 <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gerrit"/>
-<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Dgerrit.plugin-classes=${resource_loc:/gerrit/buck-out}/eclipse/plugins"/>
+<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Dgerrit.plugin-classes=${resource_loc:/gerrit/eclipse-out}/plugins"/>
 </launchConfiguration>

tools/eclipse/gerrit_gwt_debug.launch

@@ -16,7 +16,7 @@
 </listAttribute>
 <booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
 <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.google.gerrit.gwtdebug.GerritGwtDebugLauncher"/>
-<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-strict -noprecompile -src ${resource_loc:/gerrit} -workDir ${resource_loc:/gerrit}/buck-out/gen/gerrit-gwtui com.google.gerrit.GerritGwtUI -src ${resource_loc:/gerrit}/gerrit-plugin-gwtui/src/main/java -- --console-log --show-stack-trace -d ${resource_loc:/gerrit}/../gerrit_testsite"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-strict -noprecompile -src ${resource_loc:/gerrit} -workDir ${resource_loc:/gerrit}/.gwt_work_dir com.google.gerrit.GerritGwtUI -src ${resource_loc:/gerrit}/gerrit-plugin-gwtui/src/main/java -- --console-log --show-stack-trace -d ${resource_loc:/gerrit}/../gerrit_testsite"/>
 <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gerrit"/>
 <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xmx1024M&#10;-XX:MaxPermSize=256M&#10;-Dgerrit.disable-gwtui-recompile=true"/>
 </launchConfiguration>

tools/eclipse/project.py

@@ -75,6 +75,10 @@ def gen_project(name='gerrit', root=ROOT):
 </projectDescription>\
 """, file=fd)
 
+def gen_primary_build_tool():
+  with open(path.join(ROOT, ".primary_build_tool"), 'w') as fd:
+    fd.write("buck")
+
 def gen_plugin_classpath(root):
   p = path.join(root, '.classpath')
   with open(p, 'w') as fd:
@@ -244,6 +248,12 @@ try:
   gen_project(args.project_name)
   gen_classpath()
   gen_factorypath()
+  gen_primary_build_tool()
+
+  # TODO(davido): Remove this when GWT gone
+  gwt_working_dir = ".gwt_work_dir"
+  if not path.isdir(gwt_working_dir):
+    makedirs(path.join(ROOT, gwt_working_dir))
 
   try:
     targets = ['//tools:buck'] + MAIN + GWT

tools/eclipse/project_bzl.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# TODO(sop): Remove hack after Buck supports Eclipse
+
+from __future__ import print_function
+# TODO(davido): use Google style for importing instead:
+# import optparse
+# ...
+# optparse.OptionParser
+from optparse import OptionParser
+from os import environ, path, makedirs
+from subprocess import Popen, PIPE, CalledProcessError, check_call, check_output
+from xml.dom import minidom
+import re
+import sys
+
+MAIN = '//tools/eclipse:classpath'
+GWT = '//gerrit-gwtui:ui_module'
+AUTO = '//lib/auto:auto-value'
+JRE = '/'.join([
+  'org.eclipse.jdt.launching.JRE_CONTAINER',
+  'org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType',
+  'JavaSE-1.8',
+])
+# Map of targets to corresponding classpath collector rules
+cp_targets = {
+  AUTO: '//tools/eclipse:autovalue_classpath_collect',
+  GWT: '//tools/eclipse:gwt_classpath_collect',
+  MAIN: '//tools/eclipse:main_classpath_collect',
+}
+
+ROOT = path.abspath(__file__)
+while not path.exists(path.join(ROOT, 'WORKSPACE')):
+  ROOT = path.dirname(ROOT)
+
+opts = OptionParser()
+opts.add_option('--plugins', help='create eclipse projects for plugins',
+                action='store_true')
+opts.add_option('--name', help='name of the generated project',
+                action='store', default='gerrit', dest='project_name')
+args, _ = opts.parse_args()
+
+def retrieve_ext_location():
+  return check_output(['bazel', 'info', 'output_base']).strip()
+
+def gen_primary_build_tool():
+  bazel = check_output(['which', 'bazel']).strip()
+  with open(path.join(ROOT, ".primary_build_tool"), 'w') as fd:
+    fd.write("bazel=%s\n" % bazel)
+    fd.write("PATH=%s\n" % environ["PATH"])
+
+def _query_classpath(target):
+  deps = []
+  t = cp_targets[target]
+  try:
+    check_call(['bazel', 'build', t])
+  except CalledProcessError as err:
+    exit(1)
+  name = 'bazel-bin/tools/eclipse/' + t.split(':')[1] + '.runtime_classpath'
+  deps = [line.rstrip('\n') for line in open(name)]
+  return deps
+
+def gen_project(name='gerrit', root=ROOT):
+  p = path.join(root, '.project')
+  with open(p, 'w') as fd:
+    print("""\
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+  <name>%(name)s</name>
+  <buildSpec>
+    <buildCommand>
+      <name>org.eclipse.jdt.core.javabuilder</name>
+    </buildCommand>
+  </buildSpec>
+  <natures>
+    <nature>org.eclipse.jdt.core.javanature</nature>
+  </natures>
+</projectDescription>\
+    """ % {"name": name}, file=fd)
+
+def gen_plugin_classpath(root):
+  p = path.join(root, '.classpath')
+  with open(p, 'w') as fd:
+    if path.exists(path.join(root, 'src', 'test', 'java')):
+      testpath = """
+  <classpathentry excluding="**/BUILD" kind="src" path="src/test/java"\
+ out="eclipse-out/test"/>"""
+    else:
+      testpath = ""
+    print("""\
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+  <classpathentry excluding="**/BUILD" kind="src" path="src/main/java"/>%(testpath)s
+  <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+  <classpathentry combineaccessrules="false" kind="src" path="/gerrit"/>
+  <classpathentry kind="output" path="eclipse-out/classes"/>
+</classpath>""" % {"testpath": testpath}, file=fd)
+
+def gen_classpath(ext):
+  def make_classpath():
+    impl = minidom.getDOMImplementation()
+    return impl.createDocument(None, 'classpath', None)
+
+  def classpathentry(kind, path, src=None, out=None, exported=None):
+    e = doc.createElement('classpathentry')
+    e.setAttribute('kind', kind)
+    # TODO(davido): Remove this and other exclude BUILD files hack
+    # when this Bazel bug is fixed:
+    # https://github.com/bazelbuild/bazel/issues/1083
+    if kind == 'src':
+      e.setAttribute('excluding', '**/BUILD')
+    e.setAttribute('path', path)
+    if src:
+      e.setAttribute('sourcepath', src)
+    if out:
+      e.setAttribute('output', out)
+    if exported:
+      e.setAttribute('exported', 'true')
+    doc.documentElement.appendChild(e)
+
+  doc = make_classpath()
+  src = set()
+  lib = set()
+  gwt_src = set()
+  gwt_lib = set()
+  plugins = set()
+
+  # Classpath entries are absolute for cross-cell support
+  java_library = re.compile('bazel-out/local-fastbuild/bin/(.*)/[^/]+[.]jar$')
+  srcs = re.compile('(.*/external/[^/]+)/jar/(.*)[.]jar')
+  for p in _query_classpath(MAIN):
+    if p.endswith('-src.jar'):
+      # gwt_module() depends on -src.jar for Java to JavaScript compiles.
+      if p.startswith("external"):
+        p = path.join(ext, p)
+      gwt_lib.add(p)
+      continue
+
+
+    m = java_library.match(p)
+    if m:
+      src.add(m.group(1))
+      # Exceptions: both source and lib
+      if p.endswith('libquery_parser.jar') or \
+         p.endswith('prolog/libcommon.jar'):
+        lib.add(p)
+    else:
+      if p.startswith("external"):
+        p = path.join(ext, p)
+      lib.add(p)
+
+  for p in _query_classpath(GWT):
+    m = java_library.match(p)
+    if m:
+      gwt_src.add(m.group(1))
+      # Exception: we need source here for GWT SDM mode to work
+      if p.endswith('libEdit.jar'):
+        p = p[:-4] + '-src.jar'
+        assert path.exists(p), p
+        lib.add(p)
+      
+  for s in sorted(src):
+    out = None
+
+    if s.startswith('lib/'):
+      out = 'eclipse-out/lib'
+    elif s.startswith('plugins/'):
+      if args.plugins:
+        plugins.add(s)
+        continue
+      out = 'eclipse-out/' + s
+
+    p = path.join(s, 'java')
+    if path.exists(p):
+      classpathentry('src', p, out=out)
+      continue
+
+    for env in ['main', 'test']:
+      o = None
+      if out:
+        o = out + '/' + env
+      elif env == 'test':
+        o = 'eclipse-out/test'
+
+      for srctype in ['java', 'resources']:
+        p = path.join(s, 'src', env, srctype)
+        if path.exists(p):
+          classpathentry('src', p, out=o)
+
+  for libs in [lib, gwt_lib]:
+    for j in sorted(libs):
+      s = None
+      m = srcs.match(j)
+      if m:
+        prefix = m.group(1)
+        suffix = m.group(2)
+        p = path.join(prefix, "src", "%s-src.jar" % suffix)
+        if path.exists(p):
+          s = p
+      # TODO(davido): make plugins actually work
+      if args.plugins:
+        classpathentry('lib', j, s, exported=True)
+      else:
+        classpathentry('lib', j, s)
+
+  for s in sorted(gwt_src):
+    p = path.join(ROOT, s, 'src', 'main', 'java')
+    if path.exists(p):
+      classpathentry('lib', p, out='eclipse-out/gwtsrc')
+
+  classpathentry('con', JRE)
+  classpathentry('output', 'eclipse-out/classes')
+
+  p = path.join(ROOT, '.classpath')
+  with open(p, 'w') as fd:
+    doc.writexml(fd, addindent='\t', newl='\n', encoding='UTF-8')
+
+  if args.plugins:
+    for plugin in plugins:
+      plugindir = path.join(ROOT, plugin)
+      try:
+        gen_project(plugin.replace('plugins/', ""), plugindir)
+        gen_plugin_classpath(plugindir)
+      except (IOError, OSError) as err:
+        print('error generating project for %s: %s' % (plugin, err),
+              file=sys.stderr)
+
+def gen_factorypath(ext):
+  doc = minidom.getDOMImplementation().createDocument(None, 'factorypath', None)
+  for jar in _query_classpath(AUTO):
+    e = doc.createElement('factorypathentry')
+    e.setAttribute('kind', 'EXTJAR')
+    e.setAttribute('id', path.join(ext, jar))
+    e.setAttribute('enabled', 'true')
+    e.setAttribute('runInBatchMode', 'false')
+    doc.documentElement.appendChild(e)
+
+  p = path.join(ROOT, '.factorypath')
+  with open(p, 'w') as fd:
+    doc.writexml(fd, addindent='\t', newl='\n', encoding='UTF-8')
+
+try:
+  ext_location = retrieve_ext_location()
+  gen_project(args.project_name)
+  gen_classpath(ext_location)
+  gen_factorypath(ext_location)
+  gen_primary_build_tool()
+  
+  # TODO(davido): Remove this when GWT gone
+  gwt_working_dir = ".gwt_work_dir"
+  if not path.isdir(gwt_working_dir):
+    makedirs(path.join(ROOT, gwt_working_dir))
+
+  try:
+    check_call(['bazel', 'build', MAIN, GWT])
+  except CalledProcessError as err:
+    exit(1)
+except KeyboardInterrupt:
+  print('Interrupted by user', file=sys.stderr)
+  exit(1)