Add node modules license map support to license-map.py
Change-Id: Ie472da005a2b37ca149a051028922a49d2744091
This commit is contained in:
Dmitrii Filippov
parent
047240b40f
commit
daee7fccc3
@ -42,6 +42,9 @@ filegroup(
|
||||
|
||||
license_map(
|
||||
name = "licenses",
|
||||
json_maps = [
|
||||
"//polygerrit-ui/app/node_modules_licenses:polygerrit-licenses.json",
|
||||
],
|
||||
opts = ["--asciidoctor"],
|
||||
targets = [
|
||||
"//polygerrit-ui/app:polygerrit_ui",
|
||||
@ -51,6 +54,9 @@ license_map(
|
||||
|
||||
license_map(
|
||||
name = "js_licenses",
|
||||
json_maps = [
|
||||
"//polygerrit-ui/app/node_modules_licenses:polygerrit-licenses.json",
|
||||
],
|
||||
targets = [
|
||||
"//polygerrit-ui/app:polygerrit_ui",
|
||||
],
|
||||
|
||||
@ -339,6 +339,7 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
----
|
||||
|
||||
|
||||
@ -503,6 +504,5 @@ this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
----
|
||||
|
||||
|
||||
@ -2037,6 +2037,7 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
----
|
||||
|
||||
|
||||
@ -3411,7 +3412,6 @@ this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
----
|
||||
|
||||
|
||||
|
||||
@ -3,28 +3,73 @@
|
||||
# reads bazel query XML files, to join target names with their licenses.
|
||||
|
||||
from __future__ import print_function
|
||||
from collections import namedtuple
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from shutil import copyfileobj
|
||||
from sys import stdout, stderr
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
DO_NOT_DISTRIBUTE = "//lib:LICENSE-DO_NOT_DISTRIBUTE"
|
||||
|
||||
LICENSE_PREFIX = "//lib:LICENSE-"
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--asciidoctor", action="store_true")
|
||||
parser.add_argument("--json-map", action="append", dest="json_maps")
|
||||
parser.add_argument("xmls", nargs="+")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def read_file(filename):
|
||||
"Reads file and returns its content"
|
||||
with open(filename) as fd:
|
||||
return fd.read()
|
||||
|
||||
# List of files in package to which license is applied.
|
||||
# kind - enum, one of
|
||||
# AllFiles - license applies to all files in package
|
||||
# OnlySpecificFiles - license applies to all files from "files" list
|
||||
# AllFilesExceptSpecific - license applies to all files in package
|
||||
# except several files. "files" contains list of exceptions
|
||||
# files - defines list of files for the following kinds:
|
||||
# OnlySpecificFiles, AllFilesExceptSpecific.
|
||||
# Each item is a string, but not necessary a real name of a file.
|
||||
# It can be any string, understandable by human (like directory name)
|
||||
LicensedFiles = namedtuple("LicensedFiles", ["kind", "files"])
|
||||
|
||||
# PackageInfo - contains information about pacakge/files in packages to
|
||||
# which license is applied.
|
||||
# name - name of the package, as specified in package.json file
|
||||
# version - optional package version. Exists only if different versions
|
||||
# of the same package have different licenses
|
||||
# licensed_files - instance of LicensedFiles
|
||||
PackageInfo = namedtuple("PackageInfo", ["name", "version", "licensed_files"])
|
||||
|
||||
# LicenseMapItem - describe one type of license and a list of packages
|
||||
# under this license
|
||||
# name - name of the license
|
||||
# safename - name which is safe to use as an asciidoc bookmark name
|
||||
# packages - list of PackageInfo
|
||||
# license_text - license text as string
|
||||
LicenseMapItem = namedtuple("LicenseMapItem",
|
||||
["name", "safename", "packages", "license_text"])
|
||||
|
||||
|
||||
def load_xmls(xml_filenames):
|
||||
"""Load xml files produced by bazel query
|
||||
and converts them to a list of LicenseMapItem
|
||||
|
||||
Args:
|
||||
xml_filenames: list of string; each string is a filename
|
||||
Returns:
|
||||
list of LicenseMapItem
|
||||
"""
|
||||
entries = defaultdict(list)
|
||||
graph = defaultdict(list)
|
||||
handled_rules = []
|
||||
|
||||
for xml in args.xmls:
|
||||
handled_rules = set()
|
||||
for xml in xml_filenames:
|
||||
tree = ET.parse(xml)
|
||||
root = tree.getroot()
|
||||
|
||||
@ -34,7 +79,7 @@ for xml in args.xmls:
|
||||
# already handled in other xml files
|
||||
continue
|
||||
|
||||
handled_rules.append(rule_name)
|
||||
handled_rules.add(rule_name)
|
||||
for c in list(child):
|
||||
if c.tag != "rule-input":
|
||||
continue
|
||||
@ -50,6 +95,43 @@ if len(graph[DO_NOT_DISTRIBUTE]):
|
||||
print(target, file=stderr)
|
||||
exit(1)
|
||||
|
||||
result = []
|
||||
for n in sorted(graph.keys()):
|
||||
if len(graph[n]) == 0:
|
||||
continue
|
||||
|
||||
name = n[len(LICENSE_PREFIX):]
|
||||
safename = name.replace(".", "_")
|
||||
packages_names = []
|
||||
for d in sorted(graph[n]):
|
||||
if d.startswith("//lib:") or d.startswith("//lib/"):
|
||||
p = d[len("//lib:"):]
|
||||
else:
|
||||
p = d[d.index(":") + 1:].lower()
|
||||
if "__" in p:
|
||||
p = p[:p.index("__")]
|
||||
packages_names.append(p)
|
||||
|
||||
filename = n[2:].replace(":", "/")
|
||||
content = read_file(filename)
|
||||
result.append(LicenseMapItem(
|
||||
name=name,
|
||||
safename=safename,
|
||||
license_text=content,
|
||||
packages=[PackageInfo(name=name, version=None,
|
||||
licensed_files=LicensedFiles(kind="All",
|
||||
files=[])) for
|
||||
name
|
||||
in packages_names]
|
||||
)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def main():
|
||||
xml_data = load_xmls(args.xmls)
|
||||
json_map_data = load_jsons(args.json_maps)
|
||||
|
||||
if args.asciidoctor:
|
||||
# We don't want any blank line before "= Gerrit Code Review - Licenses"
|
||||
print("""= Gerrit Code Review - Licenses
|
||||
@ -93,34 +175,28 @@ updates of mirror servers, or realtime backups.
|
||||
== Licenses
|
||||
""")
|
||||
|
||||
for n in sorted(graph.keys()):
|
||||
if len(graph[n]) == 0:
|
||||
continue
|
||||
|
||||
name = n[len(LICENSE_PREFIX):]
|
||||
safename = name.replace(".", "_")
|
||||
for data in xml_data + json_map_data:
|
||||
name = data.name
|
||||
safename = data.safename
|
||||
print()
|
||||
print("[[%s]]" % safename)
|
||||
print(name)
|
||||
print()
|
||||
for d in sorted(graph[n]):
|
||||
if d.startswith("//lib:") or d.startswith("//lib/"):
|
||||
p = d[len("//lib:"):]
|
||||
else:
|
||||
p = d[d.index(":")+1:].lower()
|
||||
if "__" in p:
|
||||
p = p[:p.index("__")]
|
||||
print("* " + p)
|
||||
for p in data.packages:
|
||||
package_notice = ""
|
||||
if p.licensed_files.kind == "OnlySpecificFiles":
|
||||
package_notice = " - only the following file(s):"
|
||||
elif p.licensed_files.kind == "AllFilesExceptSpecific":
|
||||
package_notice = " - except the following file(s):"
|
||||
|
||||
print("* " + get_package_display_name(p) + package_notice)
|
||||
for file in p.licensed_files.files:
|
||||
print("** " + file)
|
||||
print()
|
||||
print("[[%s_license]]" % safename)
|
||||
print("----")
|
||||
filename = n[2:].replace(":", "/")
|
||||
try:
|
||||
with open(filename, errors='ignore') as fd:
|
||||
copyfileobj(fd, stdout)
|
||||
except TypeError:
|
||||
with open(filename) as fd:
|
||||
copyfileobj(fd, stdout)
|
||||
license_text = data.license_text
|
||||
print(data.license_text.rstrip("\r\n"))
|
||||
print()
|
||||
print("----")
|
||||
print()
|
||||
@ -131,3 +207,102 @@ GERRIT
|
||||
------
|
||||
Part of link:index.html[Gerrit Code Review]
|
||||
""")
|
||||
|
||||
def load_jsons(json_filenames):
|
||||
"""Loads information about licenses from jsons files.
|
||||
The json files are generated by license-map-generator.ts tool
|
||||
|
||||
Args:
|
||||
json_filenames: list of string; each string is a filename
|
||||
Returns:
|
||||
list of LicenseMapItem
|
||||
"""
|
||||
result = []
|
||||
for json_map in json_filenames:
|
||||
with open(json_map, 'r') as f:
|
||||
licenses_list = json.load(f)
|
||||
for license_id, license in licenses_list.items():
|
||||
name = license["licenseName"]
|
||||
safename = name.replace(".", "_")
|
||||
packages = []
|
||||
for p in license["packages"]:
|
||||
package = PackageInfo(name=p["name"], version=p["version"],
|
||||
licensed_files=get_licensed_files(
|
||||
p["licensedFiles"]))
|
||||
packages.append(package)
|
||||
result.append(LicenseMapItem(
|
||||
name=name,
|
||||
safename=safename,
|
||||
license_text=license["licenseText"],
|
||||
packages=sorted(remove_duplicated_packages(packages),
|
||||
key=lambda package: get_package_display_name(
|
||||
package)),
|
||||
))
|
||||
return result
|
||||
|
||||
def get_licensed_files(json_licensed_file_dict):
|
||||
"""Convert json dictionary to LicensedFiles"""
|
||||
kind = json_licensed_file_dict["kind"]
|
||||
if kind == "AllFiles":
|
||||
return LicensedFiles(kind="All", files=[])
|
||||
if kind == "OnlySpecificFiles" or kind == "AllFilesExceptSpecific":
|
||||
return LicensedFiles(kind=kind, files=sorted(json_licensed_file_dict["files"]))
|
||||
raise Exception("Invalid licensed files kind: %s".format(kind))
|
||||
|
||||
def get_package_display_name(package):
|
||||
"""Returns a human-readable name of package with optional version"""
|
||||
if package.version:
|
||||
return package.name + " - " + package.version
|
||||
else:
|
||||
return package.name
|
||||
|
||||
|
||||
def can_merge_packages(package_info_list):
|
||||
"""Returns true if all versions of a package can be replaced with
|
||||
a package name
|
||||
|
||||
Args:
|
||||
package_info_list: list of PackageInfo. Method assumes,
|
||||
that all items in package_info_list have the same package name,
|
||||
but different package version.
|
||||
|
||||
Returns:
|
||||
True if it is safe to print only a package name (without versions)
|
||||
False otherwise
|
||||
"""
|
||||
first = package_info_list[0]
|
||||
for package in package_info_list:
|
||||
if package.licensed_files != first.licensed_files:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def remove_duplicated_packages(package_info_list):
|
||||
""" Keep only the name of a package if all versions of the package
|
||||
have the same licensed files.
|
||||
|
||||
Args:
|
||||
package_info_list: list of PackageInfo. All items in the list
|
||||
have the same license.
|
||||
|
||||
Returns:
|
||||
list of PackageInfo with removed/replaced items.
|
||||
|
||||
Keep single version of package if all versions have the same
|
||||
license files."""
|
||||
name_to_package = defaultdict(list)
|
||||
for package in package_info_list:
|
||||
name_to_package[package.name].append(package)
|
||||
|
||||
result = []
|
||||
for package_name, packages in name_to_package.items():
|
||||
if can_merge_packages(packages):
|
||||
package = packages[0]
|
||||
result.append(PackageInfo(name=package.name, version=None,
|
||||
licensed_files=package.licensed_files))
|
||||
else:
|
||||
result.extend(packages)
|
||||
return result
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,8 +1,26 @@
|
||||
"""This file contains rules to generate and test the license map"""
|
||||
|
||||
def normalize_target_name(target):
|
||||
return target.replace("//", "").replace("/", "__").replace(":", "___")
|
||||
|
||||
def license_map(name, targets = [], opts = [], **kwargs):
|
||||
"""Generate XML for all targets that depend directly on a LICENSE file"""
|
||||
def license_map(name, targets = [], opts = [], json_maps = [], **kwargs):
|
||||
"""Generate text represantation for pacakges' and libs' licenses
|
||||
|
||||
Args:
|
||||
name: of the rule
|
||||
targets: list of targets for which licenses should be added the output file.
|
||||
The list must not include targets, for which json_map is passed in json_maps parameter
|
||||
opts: command line options for license-map.py tool
|
||||
json_maps: list of json files. Such files can be produced by node_modules_licenses rule
|
||||
for node_modules licenses.
|
||||
**kwargs: Args passed through to genrule
|
||||
|
||||
Generate: text file with the name
|
||||
gen_license_txt_{name}
|
||||
|
||||
"""
|
||||
|
||||
# Generate XML for all targets that depend directly on a LICENSE file
|
||||
xmls = []
|
||||
tools = ["//tools/bzl:license-map.py", "//lib:all-licenses"]
|
||||
for target in targets:
|
||||
@ -22,10 +40,17 @@ def license_map(name, targets = [], opts = [], **kwargs):
|
||||
opts = ["--output=xml"],
|
||||
)
|
||||
|
||||
# Add all files from the json_maps list to license-map.py command-line arguments
|
||||
json_maps_locations = []
|
||||
|
||||
for json_map in json_maps:
|
||||
json_maps_locations.append("--json-map=$(location %s)" % json_map)
|
||||
tools.append(json_map)
|
||||
|
||||
# post process the XML into our favorite format.
|
||||
native.genrule(
|
||||
name = "gen_license_txt_" + name,
|
||||
cmd = "python $(location //tools/bzl:license-map.py) %s %s > $@" % (" ".join(opts), " ".join(xmls)),
|
||||
cmd = "python $(location //tools/bzl:license-map.py) %s %s %s > $@" % (" ".join(opts), " ".join(json_maps_locations), " ".join(xmls)),
|
||||
outs = [name + ".gen.txt"],
|
||||
tools = tools,
|
||||
**kwargs
|
||||
|
||||
Loading…
Reference in New Issue
Block a user