From 9d0e8eac3b70cf1d577cfd5752966ce30ccbe5ac Mon Sep 17 00:00:00 2001 From: Yuxuan 'fishy' Wang Date: Tue, 9 Aug 2016 13:55:47 -0700 Subject: [PATCH] bazel: generate licenses.txt Modify license_map rule to: - Accept more than 1 targets - Add '--asciidoctor' option to generate asciidoctor txt file - Add support for multiple licenses for a single target (diffy_logo) Also add test_license.sh for license_test rule to handle special DO_NOT_DISTRIBUTE exceptions. TESTED: bazel build Documentation:licenses.txt bazel test gerrit-pgm:pgm_license_test bazel test gerrit-gwtui:ui_module_license_test Change-Id: Ic64b227fea34882721e6e064b1520cd9a4d5d4a4 --- Documentation/BUILD | 8 +- gerrit-gwtui/BUILD | 6 ++ gerrit-pgm/BUILD | 3 +- lib/BUILD | 6 ++ tools/bzl/BUILD | 4 +- tools/bzl/license-map.py | 151 ++++++++++++++++++++++++++++++++++---- tools/bzl/license.bzl | 84 +++++++++++---------- tools/bzl/test_license.sh | 16 ++++ 8 files changed, 222 insertions(+), 56 deletions(-) create mode 100755 tools/bzl/test_license.sh diff --git a/Documentation/BUILD b/Documentation/BUILD index 98b3ce45ac..f9bc1b2f28 100644 --- a/Documentation/BUILD +++ b/Documentation/BUILD @@ -2,6 +2,10 @@ load("//tools/bzl:license.bzl", "license_map") license_map( - name = "pgm-licenses", - target = "//gerrit-pgm:pgm", + name = "licenses", + targets = [ + "//gerrit-pgm:pgm", + "//gerrit-gwtui:ui_module", + ], + opts = ["--asciidoctor"], ) diff --git a/gerrit-gwtui/BUILD b/gerrit-gwtui/BUILD index c223885544..cc5da126d5 100644 --- a/gerrit-gwtui/BUILD +++ b/gerrit-gwtui/BUILD @@ -1,5 +1,6 @@ load('//tools/bzl:gwt.bzl', 'gwt_module') load('//tools/bzl:genrule2.bzl', 'genrule2') +load('//tools/bzl:license.bzl', 'license_test') load(':gwt.bzl', 'gwt_binary', 'gwt_genrule', 'gen_ui_module') gwt_genrule() @@ -7,3 +8,8 @@ gwt_genrule('_r') gen_ui_module(name = 'ui_module') gen_ui_module(name = 'ui_module', suffix = '_r') + +license_test( + name = "ui_module_license_test", + target = ":ui_module", +) diff --git a/gerrit-pgm/BUILD b/gerrit-pgm/BUILD index 895b5f112c..5bdc8fb2ee 100644 --- a/gerrit-pgm/BUILD +++ b/gerrit-pgm/BUILD @@ -164,4 +164,5 @@ junit_tests( license_test( name = "pgm_license_test", - target = ":pgm") + target = ":pgm", +) diff --git a/lib/BUILD b/lib/BUILD index 3290391ba2..fd25243985 100644 --- a/lib/BUILD +++ b/lib/BUILD @@ -36,6 +36,12 @@ exports_files([ "LICENSE-DO_NOT_DISTRIBUTE", ]) +filegroup( + name = 'all-licenses', + srcs = glob(['LICENSE-*'], exclude = ['LICENSE-DO_NOT_DISTRIBUTE']), + visibility = ['//visibility:public'], +) + java_library( name = 'servlet-api-3_1', neverlink = 1, diff --git a/tools/bzl/BUILD b/tools/bzl/BUILD index 01ae92c333..bfbbd21d7b 100644 --- a/tools/bzl/BUILD +++ b/tools/bzl/BUILD @@ -1,4 +1,6 @@ exports_files([ "license-map.py", - "test_empty.sh"]) + "test_empty.sh", + "test_license.sh", +]) diff --git a/tools/bzl/license-map.py b/tools/bzl/license-map.py index 72c7ae8277..8469f4f28a 100644 --- a/tools/bzl/license-map.py +++ b/tools/bzl/license-map.py @@ -1,25 +1,146 @@ #!/usr/bin/env python -# reads a bazel query XML file, to join target names with their licenses. +# reads bazel query XML files, to join target names with their licenses. -import sys +from __future__ import print_function + +import argparse +from collections import defaultdict +from shutil import copyfileobj +from sys import stdout, stderr import xml.etree.ElementTree as ET -tree = ET.parse(sys.argv[1]) -root = tree.getroot() +KNOWN_PROVIDED_DEPS = [ + "//lib/bouncycastle:bcpg", + "//lib/bouncycastle:bcpkix", + "//lib/bouncycastle:bcprov", +] -entries = {} +DO_NOT_DISTRIBUTE = "//lib:LICENSE-DO_NOT_DISTRIBUTE" -for child in root: - rule_name = child.attrib["name"] - for c in child.getchildren(): - if c.tag != "rule-input": +LICENSE_PREFIX = "//lib:LICENSE-" + +parser = argparse.ArgumentParser() +parser.add_argument("--asciidoctor", action="store_true") +parser.add_argument("xmls", nargs="+") +args = parser.parse_args() + +entries = defaultdict(list) +graph = defaultdict(list) +handled_rules = [] + +for xml in args.xmls: + tree = ET.parse(xml) + root = tree.getroot() + + for child in root: + rule_name = child.attrib["name"] + if rule_name in handled_rules: + # already handled in other xml files continue - license_name = c.attrib["name"] - if "//lib:LICENSE" in license_name: - assert rule_name not in entries, (license_name, entries[rule_name]) - entries[rule_name] = license_name + handled_rules.append(rule_name) + for c in child.getchildren(): + if c.tag != "rule-input": + continue -for k, v in sorted(entries.items()): - print k, v + license_name = c.attrib["name"] + if LICENSE_PREFIX in license_name: + if rule_name in KNOWN_PROVIDED_DEPS: + continue + + entries[rule_name].append(license_name) + graph[license_name].append(rule_name) + +if len(graph[DO_NOT_DISTRIBUTE]): + print("DO_NOT_DISTRIBUTE license found in:", file=stderr) + for target in graph[DO_NOT_DISTRIBUTE]: + print(target, file=stderr) + exit(1) + +if args.asciidoctor: + print( +# We don't want any blank line before "= Gerrit Code Review - Licenses" +"""= Gerrit Code Review - Licenses + +Gerrit open source software is licensed under the <>. Executable distributions also include other software +components that are provided under additional licenses. + +[[cryptography]] +== Cryptography Notice + +This distribution includes cryptographic software. The country +in which you currently reside may have restrictions on the import, +possession, use, and/or re-export to another country, of encryption +software. BEFORE using any encryption software, please check +your country's laws, regulations and policies concerning the +import, possession, or use, and re-export of encryption software, +to see if this is permitted. See the +link:http://www.wassenaar.org/[Wassenaar Arrangement] +for more information. + +The U.S. Government Department of Commerce, Bureau of Industry +and Security (BIS), has classified this software as Export +Commodity Control Number (ECCN) 5D002.C.1, which includes +information security software using or performing cryptographic +functions with asymmetric algorithms. The form and manner of +this distribution makes it eligible for export under the License +Exception ENC Technology Software Unrestricted (TSU) exception +(see the BIS Export Administration Regulations, Section 740.13) +for both object code and source code. + +Gerrit includes an SSH daemon (Apache SSHD), to support authenticated +uploads of changes directly from `git push` command line clients. + +Gerrit includes an SSH client (JSch), to support authenticated +replication of changes to remote systems, such as for automatic +updates of mirror servers, or realtime backups. + +For either feature to function, Gerrit requires the +link:http://java.sun.com/javase/technologies/security/[Java Cryptography extensions] +and/or the +link:http://www.bouncycastle.org/java.html[Bouncy Castle Crypto API] +to be installed by the end-user. + +== Licenses +""") + + for n in sorted(graph.keys()): + if len(graph[n]) == 0: + continue + + name = n[len(LICENSE_PREFIX):] + safename = name.replace(".", "_") + print() + print("[[%s]]" % safename) + print("=== " + name) + print() + for d in sorted(graph[n]): + if d.startswith("//lib:") or d.startswith("//lib/"): + p = d[len("//lib:"):] + else: + p = d[d.index(":")+1:].lower() + if "__" in p: + p = p[:p.index("__")] + print("* " + p) + print() + print("[[%s_license]]" % safename) + print("----") + with open(n[2:].replace(":", "/")) as fd: + copyfileobj(fd, stdout) + print() + print("----") + print() + + print( +""" +GERRIT +------ +Part of link:index.html[Gerrit Code Review] +""") + +else: + for k, vs in sorted(entries.items()): + for v in vs: + print(k, v) diff --git a/tools/bzl/license.bzl b/tools/bzl/license.bzl index ca6443869b..37cc70c6b8 100644 --- a/tools/bzl/license.bzl +++ b/tools/bzl/license.bzl @@ -1,47 +1,57 @@ -def license_map(name, target): - """Generate XML for all targets that depend directly on a LICENSE file""" +def normalize_target_name(target): + return target.replace("//", "").replace("/", "__").replace(":", "___") + +def license_map(name, targets = [], opts = []): + """Generate XML for all targets that depend directly on a LICENSE file""" + xmls = [] + tools = [ "//tools/bzl:license-map.py", "//lib:all-licenses" ] + for target in targets: + subname = name + "_" + normalize_target_name(target) + ".xml" + xmls.append("$(location :%s)" % subname) + tools.append(subname) native.genquery( - name = name + ".xml", - scope = [ target, ], + name = subname, + scope = [ target ], - # Find everything that depends on a license file, but remove - # the license files themselves from this list. - expression = 'rdeps(%s, filter("//lib:LICENSE.*", deps(%s)),1) - filter("//lib:LICENSE.*", deps(%s))' % (target, target, target), + # Find everything that depends on a license file, but remove + # the license files themselves from this list. + expression = 'rdeps(%s, filter("//lib:LICENSE.*", deps(%s)),1) - filter("//lib:LICENSE.*", deps(%s))' % (target, target, target), - # We are interested in the edges of the graph ({java_library, - # license-file} tuples). 'query' provides this in the XML output. - opts = [ "--output=xml"], + # We are interested in the edges of the graph ({java_library, + # license-file} tuples). 'query' provides this in the XML output. + opts = [ "--output=xml", ], ) - # post process the XML into our favorite format. - native.genrule( - name = "gen_license_txt_" + name, - cmd = "python $(location //tools/bzl:license-map.py) $(location :%s.xml) > $@" % name, - outs = [ name + ".txt",], - tools = [ "//tools/bzl:license-map.py", name + ".xml"]) + # post process the XML into our favorite format. + native.genrule( + name = "gen_license_txt_" + name, + cmd = "python $(location //tools/bzl:license-map.py) %s %s > $@" % (" ".join(opts), " ".join(xmls)), + outs = [ name + ".txt" ], + tools = tools + ) def license_test(name, target): - """Generate XML for all targets that depend directly on a LICENSE file""" - txt = name + "-forbidden.txt" + """Make sure a target doesn't depend on DO_NOT_DISTRIBUTE license""" + txt = name + "-forbidden.txt" - # fully qualify target name. - if target[0] not in ":/": - target = ":" + target - if target[0] != "/": - target = "//" + PACKAGE_NAME + target + # fully qualify target name. + if target[0] not in ":/": + target = ":" + target + if target[0] != "/": + target = "//" + PACKAGE_NAME + target - forbidden = "//lib:LICENSE-DO_NOT_DISTRIBUTE" - native.genquery( - name = txt, - scope = [ target, forbidden ], - # Find everything that depends on a license file, but remove - # the license files themselves from this list. - expression = 'rdeps(%s, "%s", 1) - rdeps(%s, "%s", 0)' % (target, forbidden, target, forbidden), - ) - native.sh_test( - name = name, - srcs = [ "//tools/bzl:test_empty.sh" ], - args = [ "$(location :%s)" % txt], - data = [ txt ], - ) + forbidden = "//lib:LICENSE-DO_NOT_DISTRIBUTE" + native.genquery( + name = txt, + scope = [ target, forbidden ], + # Find everything that depends on a license file, but remove + # the license files themselves from this list. + expression = 'rdeps(%s, "%s", 1) - rdeps(%s, "%s", 0)' % (target, forbidden, target, forbidden), + ) + native.sh_test( + name = name, + srcs = [ "//tools/bzl:test_license.sh" ], + args = [ "$(location :%s)" % txt ], + data = [ txt ], + ) diff --git a/tools/bzl/test_license.sh b/tools/bzl/test_license.sh new file mode 100755 index 0000000000..6ac6dabf58 --- /dev/null +++ b/tools/bzl/test_license.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +filtered="$1.filtered" + +cat $1 \ + | grep -v "//lib/bouncycastle:bcpg" \ + | grep -v "//lib/bouncycastle:bcpkix" \ + | grep -v "//lib/bouncycastle:bcprov" \ + > $filtered + +if test -s $filtered +then + echo "$filtered not empty:" + cat $filtered + exit 1 +fi