From daee7fccc3bbff3e0572a18ccea5acc33dc43d8a Mon Sep 17 00:00:00 2001 From: Dmitrii Filippov Date: Tue, 14 Jan 2020 20:42:56 +0100 Subject: [PATCH] Add node modules license map support to license-map.py Change-Id: Ie472da005a2b37ca149a051028922a49d2744091 --- Documentation/BUILD | 6 + Documentation/js_licenses.txt | 2 +- Documentation/licenses.txt | 2 +- tools/bzl/license-map.py | 295 +++++++++++++++++++++++++++------- tools/bzl/license.bzl | 31 +++- 5 files changed, 271 insertions(+), 65 deletions(-) diff --git a/Documentation/BUILD b/Documentation/BUILD index 52ab7a88a8..11d3efaa13 100644 --- a/Documentation/BUILD +++ b/Documentation/BUILD @@ -42,6 +42,9 @@ filegroup( license_map( name = "licenses", + json_maps = [ + "//polygerrit-ui/app/node_modules_licenses:polygerrit-licenses.json", + ], opts = ["--asciidoctor"], targets = [ "//polygerrit-ui/app:polygerrit_ui", @@ -51,6 +54,9 @@ license_map( license_map( name = "js_licenses", + json_maps = [ + "//polygerrit-ui/app/node_modules_licenses:polygerrit-licenses.json", + ], targets = [ "//polygerrit-ui/app:polygerrit_ui", ], diff --git a/Documentation/js_licenses.txt b/Documentation/js_licenses.txt index c2bdfbb320..3b3b69218d 100644 --- a/Documentation/js_licenses.txt +++ b/Documentation/js_licenses.txt @@ -339,6 +339,7 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ---- @@ -503,6 +504,5 @@ this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ---- diff --git a/Documentation/licenses.txt b/Documentation/licenses.txt index f63c1b488e..fa8c878040 100644 --- a/Documentation/licenses.txt +++ b/Documentation/licenses.txt @@ -2037,6 +2037,7 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ---- @@ -3411,7 +3412,6 @@ this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ---- diff --git a/tools/bzl/license-map.py b/tools/bzl/license-map.py index 2779130f47..c32579c4a4 100644 --- a/tools/bzl/license-map.py +++ b/tools/bzl/license-map.py @@ -3,56 +3,138 @@ # reads bazel query XML files, to join target names with their licenses. from __future__ import print_function +from collections import namedtuple import argparse +import json from collections import defaultdict -from shutil import copyfileobj from sys import stdout, stderr import xml.etree.ElementTree as ET - DO_NOT_DISTRIBUTE = "//lib:LICENSE-DO_NOT_DISTRIBUTE" LICENSE_PREFIX = "//lib:LICENSE-" parser = argparse.ArgumentParser() parser.add_argument("--asciidoctor", action="store_true") +parser.add_argument("--json-map", action="append", dest="json_maps") parser.add_argument("xmls", nargs="+") args = parser.parse_args() -entries = defaultdict(list) -graph = defaultdict(list) -handled_rules = [] -for xml in args.xmls: - tree = ET.parse(xml) - root = tree.getroot() +def read_file(filename): + "Reads file and returns its content" + with open(filename) as fd: + return fd.read() - for child in root: - rule_name = child.attrib["name"] - if rule_name in handled_rules: - # already handled in other xml files - continue +# List of files in package to which license is applied. +# kind - enum, one of +# AllFiles - license applies to all files in package +# OnlySpecificFiles - license applies to all files from "files" list +# AllFilesExceptSpecific - license applies to all files in package +# except several files. "files" contains list of exceptions +# files - defines list of files for the following kinds: +# OnlySpecificFiles, AllFilesExceptSpecific. +# Each item is a string, but not necessary a real name of a file. +# It can be any string, understandable by human (like directory name) +LicensedFiles = namedtuple("LicensedFiles", ["kind", "files"]) - handled_rules.append(rule_name) - for c in list(child): - if c.tag != "rule-input": +# PackageInfo - contains information about pacakge/files in packages to +# which license is applied. +# name - name of the package, as specified in package.json file +# version - optional package version. Exists only if different versions +# of the same package have different licenses +# licensed_files - instance of LicensedFiles +PackageInfo = namedtuple("PackageInfo", ["name", "version", "licensed_files"]) + +# LicenseMapItem - describe one type of license and a list of packages +# under this license +# name - name of the license +# safename - name which is safe to use as an asciidoc bookmark name +# packages - list of PackageInfo +# license_text - license text as string +LicenseMapItem = namedtuple("LicenseMapItem", + ["name", "safename", "packages", "license_text"]) + + +def load_xmls(xml_filenames): + """Load xml files produced by bazel query + and converts them to a list of LicenseMapItem + + Args: + xml_filenames: list of string; each string is a filename + Returns: + list of LicenseMapItem + """ + entries = defaultdict(list) + graph = defaultdict(list) + handled_rules = set() + for xml in xml_filenames: + tree = ET.parse(xml) + root = tree.getroot() + + for child in root: + rule_name = child.attrib["name"] + if rule_name in handled_rules: + # already handled in other xml files continue - license_name = c.attrib["name"] - if LICENSE_PREFIX in license_name: - entries[rule_name].append(license_name) - graph[license_name].append(rule_name) + handled_rules.add(rule_name) + for c in list(child): + if c.tag != "rule-input": + continue -if len(graph[DO_NOT_DISTRIBUTE]): - print("DO_NOT_DISTRIBUTE license found in:", file=stderr) - for target in graph[DO_NOT_DISTRIBUTE]: - print(target, file=stderr) - exit(1) + license_name = c.attrib["name"] + if LICENSE_PREFIX in license_name: + entries[rule_name].append(license_name) + graph[license_name].append(rule_name) -if args.asciidoctor: - # We don't want any blank line before "= Gerrit Code Review - Licenses" - print("""= Gerrit Code Review - Licenses + if len(graph[DO_NOT_DISTRIBUTE]): + print("DO_NOT_DISTRIBUTE license found in:", file=stderr) + for target in graph[DO_NOT_DISTRIBUTE]: + print(target, file=stderr) + exit(1) + + result = [] + for n in sorted(graph.keys()): + if len(graph[n]) == 0: + continue + + name = n[len(LICENSE_PREFIX):] + safename = name.replace(".", "_") + packages_names = [] + for d in sorted(graph[n]): + if d.startswith("//lib:") or d.startswith("//lib/"): + p = d[len("//lib:"):] + else: + p = d[d.index(":") + 1:].lower() + if "__" in p: + p = p[:p.index("__")] + packages_names.append(p) + + filename = n[2:].replace(":", "/") + content = read_file(filename) + result.append(LicenseMapItem( + name=name, + safename=safename, + license_text=content, + packages=[PackageInfo(name=name, version=None, + licensed_files=LicensedFiles(kind="All", + files=[])) for + name + in packages_names] + ) + ) + + return result + +def main(): + xml_data = load_xmls(args.xmls) + json_map_data = load_jsons(args.json_maps) + + if args.asciidoctor: + # We don't want any blank line before "= Gerrit Code Review - Licenses" + print("""= Gerrit Code Review - Licenses // DO NOT EDIT - GENERATED AUTOMATICALLY. @@ -93,41 +175,134 @@ updates of mirror servers, or realtime backups. == Licenses """) -for n in sorted(graph.keys()): - if len(graph[n]) == 0: - continue + for data in xml_data + json_map_data: + name = data.name + safename = data.safename + print() + print("[[%s]]" % safename) + print(name) + print() + for p in data.packages: + package_notice = "" + if p.licensed_files.kind == "OnlySpecificFiles": + package_notice = " - only the following file(s):" + elif p.licensed_files.kind == "AllFilesExceptSpecific": + package_notice = " - except the following file(s):" - name = n[len(LICENSE_PREFIX):] - safename = name.replace(".", "_") - print() - print("[[%s]]" % safename) - print(name) - print() - for d in sorted(graph[n]): - if d.startswith("//lib:") or d.startswith("//lib/"): - p = d[len("//lib:"):] - else: - p = d[d.index(":")+1:].lower() - if "__" in p: - p = p[:p.index("__")] - print("* " + p) - print() - print("[[%s_license]]" % safename) - print("----") - filename = n[2:].replace(":", "/") - try: - with open(filename, errors='ignore') as fd: - copyfileobj(fd, stdout) - except TypeError: - with open(filename) as fd: - copyfileobj(fd, stdout) - print() - print("----") - print() + print("* " + get_package_display_name(p) + package_notice) + for file in p.licensed_files.files: + print("** " + file) + print() + print("[[%s_license]]" % safename) + print("----") + license_text = data.license_text + print(data.license_text.rstrip("\r\n")) + print() + print("----") + print() -if args.asciidoctor: - print(""" + if args.asciidoctor: + print(""" GERRIT ------ Part of link:index.html[Gerrit Code Review] """) + +def load_jsons(json_filenames): + """Loads information about licenses from jsons files. + The json files are generated by license-map-generator.ts tool + + Args: + json_filenames: list of string; each string is a filename + Returns: + list of LicenseMapItem + """ + result = [] + for json_map in json_filenames: + with open(json_map, 'r') as f: + licenses_list = json.load(f) + for license_id, license in licenses_list.items(): + name = license["licenseName"] + safename = name.replace(".", "_") + packages = [] + for p in license["packages"]: + package = PackageInfo(name=p["name"], version=p["version"], + licensed_files=get_licensed_files( + p["licensedFiles"])) + packages.append(package) + result.append(LicenseMapItem( + name=name, + safename=safename, + license_text=license["licenseText"], + packages=sorted(remove_duplicated_packages(packages), + key=lambda package: get_package_display_name( + package)), + )) + return result + +def get_licensed_files(json_licensed_file_dict): + """Convert json dictionary to LicensedFiles""" + kind = json_licensed_file_dict["kind"] + if kind == "AllFiles": + return LicensedFiles(kind="All", files=[]) + if kind == "OnlySpecificFiles" or kind == "AllFilesExceptSpecific": + return LicensedFiles(kind=kind, files=sorted(json_licensed_file_dict["files"])) + raise Exception("Invalid licensed files kind: %s".format(kind)) + +def get_package_display_name(package): + """Returns a human-readable name of package with optional version""" + if package.version: + return package.name + " - " + package.version + else: + return package.name + + +def can_merge_packages(package_info_list): + """Returns true if all versions of a package can be replaced with + a package name + + Args: + package_info_list: list of PackageInfo. Method assumes, + that all items in package_info_list have the same package name, + but different package version. + + Returns: + True if it is safe to print only a package name (without versions) + False otherwise + """ + first = package_info_list[0] + for package in package_info_list: + if package.licensed_files != first.licensed_files: + return False + return True + + +def remove_duplicated_packages(package_info_list): + """ Keep only the name of a package if all versions of the package + have the same licensed files. + + Args: + package_info_list: list of PackageInfo. All items in the list + have the same license. + + Returns: + list of PackageInfo with removed/replaced items. + + Keep single version of package if all versions have the same + license files.""" + name_to_package = defaultdict(list) + for package in package_info_list: + name_to_package[package.name].append(package) + + result = [] + for package_name, packages in name_to_package.items(): + if can_merge_packages(packages): + package = packages[0] + result.append(PackageInfo(name=package.name, version=None, + licensed_files=package.licensed_files)) + else: + result.extend(packages) + return result + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/bzl/license.bzl b/tools/bzl/license.bzl index 5a6bf7fa63..cdb13d074e 100644 --- a/tools/bzl/license.bzl +++ b/tools/bzl/license.bzl @@ -1,8 +1,26 @@ +"""This file contains rules to generate and test the license map""" + def normalize_target_name(target): return target.replace("//", "").replace("/", "__").replace(":", "___") -def license_map(name, targets = [], opts = [], **kwargs): - """Generate XML for all targets that depend directly on a LICENSE file""" +def license_map(name, targets = [], opts = [], json_maps = [], **kwargs): + """Generate text represantation for pacakges' and libs' licenses + + Args: + name: of the rule + targets: list of targets for which licenses should be added the output file. + The list must not include targets, for which json_map is passed in json_maps parameter + opts: command line options for license-map.py tool + json_maps: list of json files. Such files can be produced by node_modules_licenses rule + for node_modules licenses. + **kwargs: Args passed through to genrule + + Generate: text file with the name + gen_license_txt_{name} + + """ + + # Generate XML for all targets that depend directly on a LICENSE file xmls = [] tools = ["//tools/bzl:license-map.py", "//lib:all-licenses"] for target in targets: @@ -22,10 +40,17 @@ def license_map(name, targets = [], opts = [], **kwargs): opts = ["--output=xml"], ) + # Add all files from the json_maps list to license-map.py command-line arguments + json_maps_locations = [] + + for json_map in json_maps: + json_maps_locations.append("--json-map=$(location %s)" % json_map) + tools.append(json_map) + # post process the XML into our favorite format. native.genrule( name = "gen_license_txt_" + name, - cmd = "python $(location //tools/bzl:license-map.py) %s %s > $@" % (" ".join(opts), " ".join(xmls)), + cmd = "python $(location //tools/bzl:license-map.py) %s %s %s > $@" % (" ".join(opts), " ".join(json_maps_locations), " ".join(xmls)), outs = [name + ".gen.txt"], tools = tools, **kwargs