bazel: generate licenses.txt

Modify license_map rule to:

- Accept more than 1 targets
- Add '--asciidoctor' option to generate asciidoctor txt file
- Add support for multiple licenses for a single target (diffy_logo)

Also add test_license.sh for license_test rule to handle special
DO_NOT_DISTRIBUTE exceptions.

TESTED:
  bazel build Documentation:licenses.txt
  bazel test gerrit-pgm:pgm_license_test
  bazel test gerrit-gwtui:ui_module_license_test

Change-Id: Ic64b227fea34882721e6e064b1520cd9a4d5d4a4

Documentation/BUILD

@@ -2,6 +2,10 @@
 load("//tools/bzl:license.bzl", "license_map")
 
 license_map(
-  name = "pgm-licenses",
+  name = "licenses",
-  target = "//gerrit-pgm:pgm",
+  targets = [
+    "//gerrit-pgm:pgm",
+    "//gerrit-gwtui:ui_module",
+  ],
+  opts = ["--asciidoctor"],
 )

gerrit-gwtui/BUILD

@@ -1,5 +1,6 @@
 load('//tools/bzl:gwt.bzl', 'gwt_module')
 load('//tools/bzl:genrule2.bzl', 'genrule2')
+load('//tools/bzl:license.bzl', 'license_test')
 load(':gwt.bzl', 'gwt_binary', 'gwt_genrule', 'gen_ui_module')
 
 gwt_genrule()
@@ -7,3 +8,8 @@ gwt_genrule('_r')
 
 gen_ui_module(name = 'ui_module')
 gen_ui_module(name = 'ui_module', suffix = '_r')
+
+license_test(
+  name = "ui_module_license_test",
+  target = ":ui_module",
+)

gerrit-pgm/BUILD

@@ -164,4 +164,5 @@ junit_tests(
 
 license_test(
   name = "pgm_license_test",
-  target = ":pgm")
+  target = ":pgm",
+)

lib/BUILD

@@ -36,6 +36,12 @@ exports_files([
   "LICENSE-DO_NOT_DISTRIBUTE",
 ])
 
+filegroup(
+  name = 'all-licenses',
+  srcs = glob(['LICENSE-*'], exclude = ['LICENSE-DO_NOT_DISTRIBUTE']),
+  visibility = ['//visibility:public'],
+)
+
 java_library(
   name = 'servlet-api-3_1',
   neverlink = 1,

tools/bzl/BUILD

@@ -1,4 +1,6 @@
 
 exports_files([
   "license-map.py",
-  "test_empty.sh"])
+  "test_empty.sh",
+  "test_license.sh",
+])

tools/bzl/license-map.py

@@ -1,25 +1,146 @@
 #!/usr/bin/env python
 
-# reads a bazel query XML file, to join target names with their licenses.
+# reads bazel query XML files, to join target names with their licenses.
 
-import sys
+from __future__ import print_function
+
+import argparse
+from collections import defaultdict
+from shutil import copyfileobj
+from sys import stdout, stderr
 import xml.etree.ElementTree as ET
 
-tree = ET.parse(sys.argv[1])
+KNOWN_PROVIDED_DEPS = [
-root = tree.getroot()
+  "//lib/bouncycastle:bcpg",
+  "//lib/bouncycastle:bcpkix",
+  "//lib/bouncycastle:bcprov",
+]
 
-entries = {}
+DO_NOT_DISTRIBUTE = "//lib:LICENSE-DO_NOT_DISTRIBUTE"
+
+LICENSE_PREFIX = "//lib:LICENSE-"
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--asciidoctor", action="store_true")
+parser.add_argument("xmls", nargs="+")
+args = parser.parse_args()
+
+entries = defaultdict(list)
+graph = defaultdict(list)
+handled_rules = []
+
+for xml in args.xmls:
+  tree = ET.parse(xml)
+  root = tree.getroot()
 
   for child in root:
     rule_name = child.attrib["name"]
+    if rule_name in handled_rules:
+      # already handled in other xml files
+      continue
+
+    handled_rules.append(rule_name)
     for c in child.getchildren():
       if c.tag != "rule-input":
         continue
 
       license_name = c.attrib["name"]
-    if "//lib:LICENSE" in license_name:
+      if LICENSE_PREFIX in license_name:
-      assert rule_name not in entries, (license_name, entries[rule_name])
+        if rule_name in KNOWN_PROVIDED_DEPS:
-      entries[rule_name] = license_name
+          continue
 
-for k, v in sorted(entries.items()):
+        entries[rule_name].append(license_name)
-  print k, v
+        graph[license_name].append(rule_name)
+
+if len(graph[DO_NOT_DISTRIBUTE]):
+  print("DO_NOT_DISTRIBUTE license found in:", file=stderr)
+  for target in graph[DO_NOT_DISTRIBUTE]:
+    print(target, file=stderr)
+  exit(1)
+
+if args.asciidoctor:
+  print(
+# We don't want any blank line before "= Gerrit Code Review - Licenses"
+"""= Gerrit Code Review - Licenses
+
+Gerrit open source software is licensed under the <<Apache2_0,Apache
+License 2.0>>.  Executable distributions also include other software
+components that are provided under additional licenses.
+
+[[cryptography]]
+== Cryptography Notice
+
+This distribution includes cryptographic software.  The country
+in which you currently reside may have restrictions on the import,
+possession, use, and/or re-export to another country, of encryption
+software.  BEFORE using any encryption software, please check
+your country's laws, regulations and policies concerning the
+import, possession, or use, and re-export of encryption software,
+to see if this is permitted.  See the
+link:http://www.wassenaar.org/[Wassenaar Arrangement]
+for more information.
+
+The U.S. Government Department of Commerce, Bureau of Industry
+and Security (BIS), has classified this software as Export
+Commodity Control Number (ECCN) 5D002.C.1, which includes
+information security software using or performing cryptographic
+functions with asymmetric algorithms.  The form and manner of
+this distribution makes it eligible for export under the License
+Exception ENC Technology Software Unrestricted (TSU) exception
+(see the BIS Export Administration Regulations, Section 740.13)
+for both object code and source code.
+
+Gerrit includes an SSH daemon (Apache SSHD), to support authenticated
+uploads of changes directly from `git push` command line clients.
+
+Gerrit includes an SSH client (JSch), to support authenticated
+replication of changes to remote systems, such as for automatic
+updates of mirror servers, or realtime backups.
+
+For either feature to function, Gerrit requires the
+link:http://java.sun.com/javase/technologies/security/[Java Cryptography extensions]
+and/or the
+link:http://www.bouncycastle.org/java.html[Bouncy Castle Crypto API]
+to be installed by the end-user.
+
+== Licenses
+""")
+
+  for n in sorted(graph.keys()):
+    if len(graph[n]) == 0:
+      continue
+
+    name = n[len(LICENSE_PREFIX):]
+    safename = name.replace(".", "_")
+    print()
+    print("[[%s]]" % safename)
+    print("=== " + name)
+    print()
+    for d in sorted(graph[n]):
+      if d.startswith("//lib:") or d.startswith("//lib/"):
+        p = d[len("//lib:"):]
+      else:
+        p = d[d.index(":")+1:].lower()
+      if "__" in p:
+        p = p[:p.index("__")]
+      print("* " + p)
+    print()
+    print("[[%s_license]]" % safename)
+    print("----")
+    with open(n[2:].replace(":", "/")) as fd:
+      copyfileobj(fd, stdout)
+    print()
+    print("----")
+    print()
+
+  print(
+"""
+GERRIT
+------
+Part of link:index.html[Gerrit Code Review]
+""")
+
+else:
+  for k, vs in sorted(entries.items()):
+    for v in vs:
+      print(k, v)

tools/bzl/license.bzl

@@ -1,9 +1,18 @@
 
-def license_map(name, target):
+def normalize_target_name(target):
+  return target.replace("//", "").replace("/", "__").replace(":", "___")
+
+def license_map(name, targets = [], opts = []):
   """Generate XML for all targets that depend directly on a LICENSE file"""
+  xmls = []
+  tools = [ "//tools/bzl:license-map.py", "//lib:all-licenses" ]
+  for target in targets:
+    subname = name + "_" + normalize_target_name(target) + ".xml"
+    xmls.append("$(location :%s)" % subname)
+    tools.append(subname)
     native.genquery(
-        name = name + ".xml",
+      name = subname,
-        scope = [ target, ],
+      scope = [ target ],
 
       # Find everything that depends on a license file, but remove
       # the license files themselves from this list.
@@ -11,18 +20,19 @@ def license_map(name, target):
 
       # We are interested in the edges of the graph ({java_library,
       # license-file} tuples).  'query' provides this in the XML output.
-        opts = [ "--output=xml"],
+      opts = [ "--output=xml", ],
     )
 
   # post process the XML into our favorite format.
   native.genrule(
     name = "gen_license_txt_" + name,
-        cmd = "python $(location //tools/bzl:license-map.py) $(location :%s.xml) > $@" % name,
+    cmd = "python $(location //tools/bzl:license-map.py) %s %s > $@" % (" ".join(opts), " ".join(xmls)),
-        outs = [ name + ".txt",],
+    outs = [ name + ".txt" ],
-        tools = [ "//tools/bzl:license-map.py", name + ".xml"])
+    tools = tools
+  )
 
 def license_test(name, target):
-    """Generate XML for all targets that depend directly on a LICENSE file"""
+  """Make sure a target doesn't depend on DO_NOT_DISTRIBUTE license"""
   txt = name + "-forbidden.txt"
 
   # fully qualify target name.
@@ -41,7 +51,7 @@ def license_test(name, target):
   )
   native.sh_test(
     name = name,
-        srcs = [ "//tools/bzl:test_empty.sh" ],
+    srcs = [ "//tools/bzl:test_license.sh" ],
     args  = [ "$(location :%s)" % txt ],
     data = [ txt ],
   )

tools/bzl/test_license.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+filtered="$1.filtered"
+
+cat $1 \
+  | grep -v "//lib/bouncycastle:bcpg" \
+  | grep -v "//lib/bouncycastle:bcpkix" \
+  | grep -v "//lib/bouncycastle:bcprov" \
+  > $filtered
+
+if test -s $filtered
+then
+  echo "$filtered not empty:"
+  cat $filtered
+  exit 1
+fi