opendev/gerrit
Code Issues Proposed changes

Add node modules license map support to license-map.py

Browse Source 
Change-Id: Ie472da005a2b37ca149a051028922a49d2744091
This commit is contained in:
Dmitrii Filippov 2020-01-14 20:42:56 +01:00
parent 047240b40f
commit daee7fccc3
5 changed files with 271 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Documentation/BUILD
View File

@ -42,6 +42,9 @@ filegroup(
license_map(
name = "licenses",
json_maps = [
"//polygerrit-ui/app/node_modules_licenses:polygerrit-licenses.json",
],
opts = ["--asciidoctor"],
targets = [
"//polygerrit-ui/app:polygerrit_ui",
@ -51,6 +54,9 @@ license_map(
license_map(
name = "js_licenses",
json_maps = [
"//polygerrit-ui/app/node_modules_licenses:polygerrit-licenses.json",
],
targets = [
"//polygerrit-ui/app:polygerrit_ui",
],

2
Documentation/js_licenses.txt
View File

@ -339,6 +339,7 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
----
@ -503,6 +504,5 @@ this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
----

2
Documentation/licenses.txt
View File

@ -2037,6 +2037,7 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
----
@ -3411,7 +3412,6 @@ this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
----

295
tools/bzl/license-map.py
View File

@ -3,56 +3,138 @@
# reads bazel query XML files, to join target names with their licenses.
from __future__ import print_function
from collections import namedtuple
import argparse
import json
from collections import defaultdict
from shutil import copyfileobj
from sys import stdout, stderr
import xml.etree.ElementTree as ET
DO_NOT_DISTRIBUTE = "//lib:LICENSE-DO_NOT_DISTRIBUTE"
LICENSE_PREFIX = "//lib:LICENSE-"
parser = argparse.ArgumentParser()
parser.add_argument("--asciidoctor", action="store_true")
parser.add_argument("--json-map", action="append", dest="json_maps")
parser.add_argument("xmls", nargs="+")
args = parser.parse_args()
entries = defaultdict(list)
graph = defaultdict(list)
handled_rules = []
for xml in args.xmls:
tree = ET.parse(xml)
root = tree.getroot()
def read_file(filename):
"Reads file and returns its content"
with open(filename) as fd:
return fd.read()
for child in root:
rule_name = child.attrib["name"]
if rule_name in handled_rules:
# already handled in other xml files
continue
# List of files in package to which license is applied.
# kind - enum, one of
# AllFiles - license applies to all files in package
# OnlySpecificFiles - license applies to all files from "files" list
# AllFilesExceptSpecific - license applies to all files in package
# except several files. "files" contains list of exceptions
# files - defines list of files for the following kinds:
# OnlySpecificFiles, AllFilesExceptSpecific.
# Each item is a string, but not necessary a real name of a file.
# It can be any string, understandable by human (like directory name)
LicensedFiles = namedtuple("LicensedFiles", ["kind", "files"])
handled_rules.append(rule_name)
for c in list(child):
if c.tag != "rule-input":
# PackageInfo - contains information about pacakge/files in packages to
# which license is applied.
# name - name of the package, as specified in package.json file
# version - optional package version. Exists only if different versions
# of the same package have different licenses
# licensed_files - instance of LicensedFiles
PackageInfo = namedtuple("PackageInfo", ["name", "version", "licensed_files"])
# LicenseMapItem - describe one type of license and a list of packages
# under this license
# name - name of the license
# safename - name which is safe to use as an asciidoc bookmark name
# packages - list of PackageInfo
# license_text - license text as string
LicenseMapItem = namedtuple("LicenseMapItem",
["name", "safename", "packages", "license_text"])
def load_xmls(xml_filenames):
"""Load xml files produced by bazel query
and converts them to a list of LicenseMapItem
Args:
xml_filenames: list of string; each string is a filename
Returns:
list of LicenseMapItem
"""
entries = defaultdict(list)
graph = defaultdict(list)
handled_rules = set()
for xml in xml_filenames:
tree = ET.parse(xml)
root = tree.getroot()
for child in root:
rule_name = child.attrib["name"]
if rule_name in handled_rules:
# already handled in other xml files
continue
license_name = c.attrib["name"]
if LICENSE_PREFIX in license_name:
entries[rule_name].append(license_name)
graph[license_name].append(rule_name)
handled_rules.add(rule_name)
for c in list(child):
if c.tag != "rule-input":
continue
if len(graph[DO_NOT_DISTRIBUTE]):
print("DO_NOT_DISTRIBUTE license found in:", file=stderr)
for target in graph[DO_NOT_DISTRIBUTE]:
print(target, file=stderr)
exit(1)
license_name = c.attrib["name"]
if LICENSE_PREFIX in license_name:
entries[rule_name].append(license_name)
graph[license_name].append(rule_name)
if args.asciidoctor:
# We don't want any blank line before "= Gerrit Code Review - Licenses"
print("""= Gerrit Code Review - Licenses
if len(graph[DO_NOT_DISTRIBUTE]):
print("DO_NOT_DISTRIBUTE license found in:", file=stderr)
for target in graph[DO_NOT_DISTRIBUTE]:
print(target, file=stderr)
exit(1)
result = []
for n in sorted(graph.keys()):
if len(graph[n]) == 0:
continue
name = n[len(LICENSE_PREFIX):]
safename = name.replace(".", "_")
packages_names = []
for d in sorted(graph[n]):
if d.startswith("//lib:") or d.startswith("//lib/"):
p = d[len("//lib:"):]
else:
p = d[d.index(":") + 1:].lower()
if "__" in p:
p = p[:p.index("__")]
packages_names.append(p)
filename = n[2:].replace(":", "/")
content = read_file(filename)
result.append(LicenseMapItem(
name=name,
safename=safename,
license_text=content,
packages=[PackageInfo(name=name, version=None,
licensed_files=LicensedFiles(kind="All",
files=[])) for
name
in packages_names]
)
)
return result
def main():
xml_data = load_xmls(args.xmls)
json_map_data = load_jsons(args.json_maps)
if args.asciidoctor:
# We don't want any blank line before "= Gerrit Code Review - Licenses"
print("""= Gerrit Code Review - Licenses
// DO NOT EDIT - GENERATED AUTOMATICALLY.
@ -93,41 +175,134 @@ updates of mirror servers, or realtime backups.
== Licenses
""")
for n in sorted(graph.keys()):
if len(graph[n]) == 0:
continue
for data in xml_data + json_map_data:
name = data.name
safename = data.safename
print()
print("[[%s]]" % safename)
print(name)
print()
for p in data.packages:
package_notice = ""
if p.licensed_files.kind == "OnlySpecificFiles":
package_notice = " - only the following file(s):"
elif p.licensed_files.kind == "AllFilesExceptSpecific":
package_notice = " - except the following file(s):"
name = n[len(LICENSE_PREFIX):]
safename = name.replace(".", "_")
print()
print("[[%s]]" % safename)
print(name)
print()
for d in sorted(graph[n]):
if d.startswith("//lib:") or d.startswith("//lib/"):
p = d[len("//lib:"):]
else:
p = d[d.index(":")+1:].lower()
if "__" in p:
p = p[:p.index("__")]
print("* " + p)
print()
print("[[%s_license]]" % safename)
print("----")
filename = n[2:].replace(":", "/")
try:
with open(filename, errors='ignore') as fd:
copyfileobj(fd, stdout)
except TypeError:
with open(filename) as fd:
copyfileobj(fd, stdout)
print()
print("----")
print()
print("* " + get_package_display_name(p) + package_notice)
for file in p.licensed_files.files:
print("** " + file)
print()
print("[[%s_license]]" % safename)
print("----")
license_text = data.license_text
print(data.license_text.rstrip("\r\n"))
print()
print("----")
print()
if args.asciidoctor:
print("""
if args.asciidoctor:
print("""
GERRIT
------
Part of link:index.html[Gerrit Code Review]
""")
def load_jsons(json_filenames):
"""Loads information about licenses from jsons files.
The json files are generated by license-map-generator.ts tool
Args:
json_filenames: list of string; each string is a filename
Returns:
list of LicenseMapItem
"""
result = []
for json_map in json_filenames:
with open(json_map, 'r') as f:
licenses_list = json.load(f)
for license_id, license in licenses_list.items():
name = license["licenseName"]
safename = name.replace(".", "_")
packages = []
for p in license["packages"]:
package = PackageInfo(name=p["name"], version=p["version"],
licensed_files=get_licensed_files(
p["licensedFiles"]))
packages.append(package)
result.append(LicenseMapItem(
name=name,
safename=safename,
license_text=license["licenseText"],
packages=sorted(remove_duplicated_packages(packages),
key=lambda package: get_package_display_name(
package)),
))
return result
def get_licensed_files(json_licensed_file_dict):
"""Convert json dictionary to LicensedFiles"""
kind = json_licensed_file_dict["kind"]
if kind == "AllFiles":
return LicensedFiles(kind="All", files=[])
if kind == "OnlySpecificFiles" or kind == "AllFilesExceptSpecific":
return LicensedFiles(kind=kind, files=sorted(json_licensed_file_dict["files"]))
raise Exception("Invalid licensed files kind: %s".format(kind))
def get_package_display_name(package):
"""Returns a human-readable name of package with optional version"""
if package.version:
return package.name + " - " + package.version
else:
return package.name
def can_merge_packages(package_info_list):
"""Returns true if all versions of a package can be replaced with
a package name
Args:
package_info_list: list of PackageInfo. Method assumes,
that all items in package_info_list have the same package name,
but different package version.
Returns:
True if it is safe to print only a package name (without versions)
False otherwise
"""
first = package_info_list[0]
for package in package_info_list:
if package.licensed_files != first.licensed_files:
return False
return True
def remove_duplicated_packages(package_info_list):
""" Keep only the name of a package if all versions of the package
have the same licensed files.
Args:
package_info_list: list of PackageInfo. All items in the list
have the same license.
Returns:
list of PackageInfo with removed/replaced items.
Keep single version of package if all versions have the same
license files."""
name_to_package = defaultdict(list)
for package in package_info_list:
name_to_package[package.name].append(package)
result = []
for package_name, packages in name_to_package.items():
if can_merge_packages(packages):
package = packages[0]
result.append(PackageInfo(name=package.name, version=None,
licensed_files=package.licensed_files))
else:
result.extend(packages)
return result
if __name__ == "__main__":
main()

31
tools/bzl/license.bzl
View File

@ -1,8 +1,26 @@
"""This file contains rules to generate and test the license map"""
def normalize_target_name(target):
return target.replace("//", "").replace("/", "__").replace(":", "___")
def license_map(name, targets = [], opts = [], **kwargs):
"""Generate XML for all targets that depend directly on a LICENSE file"""
def license_map(name, targets = [], opts = [], json_maps = [], **kwargs):
"""Generate text represantation for pacakges' and libs' licenses
Args:
name: of the rule
targets: list of targets for which licenses should be added the output file.
The list must not include targets, for which json_map is passed in json_maps parameter
opts: command line options for license-map.py tool
json_maps: list of json files. Such files can be produced by node_modules_licenses rule
for node_modules licenses.
**kwargs: Args passed through to genrule
Generate: text file with the name
gen_license_txt_{name}
"""
# Generate XML for all targets that depend directly on a LICENSE file
xmls = []
tools = ["//tools/bzl:license-map.py", "//lib:all-licenses"]
for target in targets:
@ -22,10 +40,17 @@ def license_map(name, targets = [], opts = [], **kwargs):
opts = ["--output=xml"],
)
# Add all files from the json_maps list to license-map.py command-line arguments
json_maps_locations = []
for json_map in json_maps:
json_maps_locations.append("--json-map=$(location %s)" % json_map)
tools.append(json_map)
# post process the XML into our favorite format.
native.genrule(
name = "gen_license_txt_" + name,
cmd = "python $(location //tools/bzl:license-map.py) %s %s > $@" % (" ".join(opts), " ".join(xmls)),
cmd = "python $(location //tools/bzl:license-map.py) %s %s %s > $@" % (" ".join(opts), " ".join(json_maps_locations), " ".join(xmls)),
outs = [name + ".gen.txt"],
tools = tools,
**kwargs