bazel: bower support

* npm_binary (download tarball packaged npm apps)

* bower_archive (download a zip file, to be put in WORKSPACE)

* bower_component (defining a bower library, with dependency )

* bower_component_bundle (zipping up libraries together)

* js_component (insert plain js file into bower component bundle)

* bower2bazel.py: run bower to find dependencies, generate a .bzl to
  define archives and define components

Tested:

 python tools/js/bower2bazel.py -w lib/js/bower_archives.bzl -b \
    lib/js/bower_components.bzl

 bazel build polygerrit-ui:components
 unzip -v bazel-bin/polygerrit-ui/components.zip > /tmp/baz

 buck build polygerrit-ui:polygerrit_components
 unzip -v buck-out/gen/polygerrit-ui/polygerrit_components/polygerrit_components.bower_components.zip > /tmp/buck

 diff /tmp/buck /tmp/baz

The diff corresponds to newer file versions pinned through bower2bazel.

Change-Id: I4f33914d4853bcf8afe78b4719d0e0e83b139031
This commit is contained in:
Han-Wen Nienhuys 2016-09-21 15:03:54 +02:00
parent 8178e0e3f2
commit 28e7a6d3fe
19 changed files with 1009 additions and 85 deletions

View File

@ -1,3 +1,5 @@
workspace(name="gerrit")
ANTLR_VERS = '3.5.2'
maven_jar(
@ -954,3 +956,90 @@ maven_jar(
artifact = 'org.apache.httpcomponents:httpcore-niossl:4.0-alpha6',
sha1 = '9c662e7247ca8ceb1de5de629f685c9ef3e4ab58',
)
load("//tools/bzl:js.bzl", "npm_binary", "bower_archive")
npm_binary(
name = "bower",
)
# bower_archive() seed components.
bower_archive(
name = 'iron-autogrow-textarea',
package = 'polymerelements/iron-autogrow-textarea',
version = '1.0.12',
sha1 = 'b9b6874c9a2b5be435557a827ff8bd6661672ee3',
)
bower_archive(
name = 'es6-promise',
package = 'stefanpenner/es6-promise',
version = '3.3.0',
sha1 = 'a3a797bb22132f1ef75f9a2556173f81870c2e53',
)
bower_archive(
name = 'fetch',
package = 'fetch',
version = '1.0.0',
sha1 = '1b05a2bb40c73232c2909dc196de7519fe4db7a9',
)
bower_archive(
name = 'iron-dropdown',
package = 'polymerelements/iron-dropdown',
version = '1.4.0',
sha1 = '63e3d669a09edaa31c4f05afc76b53b919ef0595',
)
bower_archive(
name = 'iron-input',
package = 'polymerelements/iron-input',
version = '1.0.10',
sha1 = '9bc0c8e81de2527125383cbcf74dd9f27e7fa9ac',
)
bower_archive(
name = 'iron-overlay-behavior',
package = 'polymerelements/iron-overlay-behavior',
version = '1.7.6',
sha1 = '83181085fda59446ce74fd0d5ca30c223f38ee4a',
)
bower_archive(
name = 'iron-selector',
package = 'polymerelements/iron-selector',
version = '1.5.2',
sha1 = 'c57235dfda7fbb987c20ad0e97aac70babf1a1bf',
)
bower_archive(
name = 'moment',
package = 'moment/moment',
version = '2.13.0',
sha1 = 'fc8ce2c799bab21f6ced7aff928244f4ca8880aa',
)
bower_archive(
name = 'page',
package = 'visionmedia/page.js',
version = '1.7.1',
sha1 = '51a05428dd4f68fae1df5f12d0e2b61ba67f7757',
)
bower_archive(
name = 'polymer',
package = 'polymer/polymer',
version = '1.4.0',
sha1 = 'b84725939ead7c7bdf9917b065f68ef8dc790d06',
)
bower_archive(
name = 'promise-polyfill',
package = 'polymerlabs/promise-polyfill',
version = '1.0.0',
sha1 = 'a3b598c06cbd7f441402e666ff748326030905d6',
)
# Bower component transitive dependencies.
load("//lib/js:bower_archives.bzl", "load_bower_archives")
load_bower_archives()

View File

@ -1,40 +1,6 @@
exports_files([
"LICENSE-antlr",
"LICENSE-Apache1.1",
"LICENSE-Apache2.0",
"LICENSE-args4j",
"LICENSE-asciidoctor",
"LICENSE-automaton",
"LICENSE-bouncycastle",
"LICENSE-CC-BY3.0-unported",
"LICENSE-clippy",
"LICENSE-codemirror-minified",
"LICENSE-codemirror-original",
"LICENSE-diffy",
"LICENSE-es6-promise",
"LICENSE-fetch",
"LICENSE-h2",
"LICENSE-highlightjs",
"LICENSE-icu4j",
"LICENSE-jgit",
"LICENSE-jsch",
"LICENSE-MPL1.1",
"LICENSE-moment",
"LICENSE-OFL1.1",
"LICENSE-ow2",
"LICENSE-page.js",
"LICENSE-polymer",
"LICENSE-postgresql",
"LICENSE-prologcafe",
"LICENSE-promise-polyfill",
"LICENSE-protobuf",
"LICENSE-PublicDomain",
"LICENSE-silk_icons",
"LICENSE-slf4j",
"LICENSE-xz",
"LICENSE-DO_NOT_DISTRIBUTE",
])
exports_files(glob([
"LICENSE-*"
]))
filegroup(
name = 'all-licenses',

34
lib/fonts/BUILD Normal file
View File

@ -0,0 +1,34 @@
load('//tools/bzl:genrule2.bzl', 'genrule2')
# Source Code Pro. Version 2.010 Roman / 1.030 Italics
# https://github.com/adobe-fonts/source-code-pro/releases/tag/2.010R-ro%2F1.030R-it
genrule2(
name = 'sourcecodepro',
cmd = 'zip -rq $@ $(SRCS)',
srcs = [
'SourceCodePro-Regular.woff',
'SourceCodePro-Regular.woff2'
],
out = 'sourcecodepro.zip',
# TODO(hanwen): fix this
# license = 'OFL1.1',
visibility = ['//visibility:public'],
)
# Open Sans at Revision 53a5266 and converted using a Google woff file
# converter (same one that Google Fonts uses).
# https://github.com/google/fonts/tree/master/apache/opensans
genrule2(
name = 'opensans',
cmd = 'zip -rq $@ $(SRCS)',
srcs = [
'OpenSans-Bold.woff',
'OpenSans-Bold.woff2',
'OpenSans-Regular.woff',
'OpenSans-Regular.woff2'
],
out = 'opensans.zip',
# TODO(hanwen): license.
# license = 'Apache2.0',
visibility = ['//visibility:public'],
)

4
lib/highlightjs/BUILD Normal file
View File

@ -0,0 +1,4 @@
exports_files([
'highlight.min.js',
])

View File

@ -124,7 +124,8 @@ def _combine_components(deps):
cmds = ['cd $TMP']
for d in deps:
cmds.append('unzip -qo $(location %s)' % d)
cmds.append('zip -r $OUT bower_components')
cmds.append("find bower_components -exec touch -t 198001010000 '{}' ';'")
cmds.append('zip -r $OUT bower_components/*')
return ' && '.join(cmds)

29
lib/js/BUILD Normal file
View File

@ -0,0 +1,29 @@
package(default_visibility = [ "//visibility:public" ])
load('//tools/bzl:genrule2.bzl', 'genrule2')
load("//tools/bzl:js.bzl", "bower_component", "js_component")
# For updating the bower versions, run
#
# python tools/js/bower2bazel.py -w lib/js/bower_archives.bzl -b lib/js/bower_components.bzl
#
# For adding a new component as dependency to a bower_component_bundle
#
# 1) add a new bower_archive in WORKSPACE
#
# 2) add bower_component(name="my_new_dependency", seed=True) here
#
# 3) run bower2bazel (see above.)
#
# 4) remove bower_component(name="my_new_dependency", .. ) here
#
load("//lib/js:bower_components.bzl", "define_bower_components")
define_bower_components()
js_component(
name = 'highlightjs',
srcs = [ "//lib/highlightjs:highlight.min.js" ],
license = '//lib:LICENSE-highlightjs',
)

68
lib/js/bower_archives.bzl Normal file
View File

@ -0,0 +1,68 @@
# DO NOT EDIT
# generated with the following command:
#
# tools/js/bower2bazel.py -w lib/js/bower_archives.bzl -b lib/js/bower_components.bzl
#
load("//tools/bzl:js.bzl", "bower_archive")
def load_bower_archives():
bower_archive(
name = "iron-a11y-announcer",
package = "iron-a11y-announcer",
version = "1.0.5",
sha1 = "007902c041dd8863a1fe893f62450852f4d8c69b")
bower_archive(
name = "iron-a11y-keys-behavior",
package = "iron-a11y-keys-behavior",
version = "1.1.9",
sha1 = "f58358ee652c67e6e721364ba50fb77a2ece1465")
bower_archive(
name = "iron-behaviors",
package = "iron-behaviors",
version = "1.0.17",
sha1 = "47df7e1c2b97978dcafa13edb50fbdb702570acd")
bower_archive(
name = "iron-fit-behavior",
package = "iron-fit-behavior",
version = "1.2.5",
sha1 = "5938815cd227843fc77ebeac480b999600a76157")
bower_archive(
name = "iron-flex-layout",
package = "iron-flex-layout",
version = "1.3.1",
sha1 = "ba696394abff5e799fc06eb11bff4720129a1b52")
bower_archive(
name = "iron-form-element-behavior",
package = "iron-form-element-behavior",
version = "1.0.6",
sha1 = "8d9e6530edc1b99bec1a5c34853911fba3701220")
bower_archive(
name = "iron-meta",
package = "iron-meta",
version = "1.1.2",
sha1 = "dc22fe05e1cb5f94f30a7193d3433ca1808773b8")
bower_archive(
name = "iron-resizable-behavior",
package = "iron-resizable-behavior",
version = "1.0.5",
sha1 = "2ebe983377dceb3794dd335131050656e23e2beb")
bower_archive(
name = "iron-validatable-behavior",
package = "iron-validatable-behavior",
version = "1.1.1",
sha1 = "480423380be0536f948735d91bc472f6e7ced5b4")
bower_archive(
name = "neon-animation",
package = "neon-animation",
version = "1.2.4",
sha1 = "e8ccbb930c4b7ff470b1450baa901618888a7fd3")
bower_archive(
name = "web-animations-js",
package = "web-animations-js",
version = "2.2.2",
sha1 = "6276a9f227da7d4ccaf77c202b50e174dd11a2c2")
bower_archive(
name = "webcomponentsjs",
package = "webcomponentsjs",
version = "0.7.22",
sha1 = "8ba97a4a279ec6973a19b171c462a7b5cf454fb9")

162
lib/js/bower_components.bzl Normal file
View File

@ -0,0 +1,162 @@
# DO NOT EDIT
# generated with the following command:
#
# tools/js/bower2bazel.py -w lib/js/bower_archives.bzl -b lib/js/bower_components.bzl
#
load("//tools/bzl:js.bzl", "bower_component")
def define_bower_components():
bower_component(
name = "es6-promise",
license = "//lib:LICENSE-polymer",
seed = True,
)
bower_component(
name = "fetch",
license = "//lib:LICENSE-fetch",
seed = True,
)
bower_component(
name = "iron-a11y-announcer",
license = "//lib:LICENSE-polymer",
deps = [ ":polymer" ],
)
bower_component(
name = "iron-a11y-keys-behavior",
license = "//lib:LICENSE-polymer",
deps = [ ":polymer" ],
)
bower_component(
name = "iron-autogrow-textarea",
license = "//lib:LICENSE-polymer",
deps = [
":iron-behaviors",
":iron-flex-layout",
":iron-form-element-behavior",
":iron-validatable-behavior",
":polymer",
],
seed = True,
)
bower_component(
name = "iron-behaviors",
license = "//lib:LICENSE-polymer",
deps = [
":iron-a11y-keys-behavior",
":polymer",
],
)
bower_component(
name = "iron-dropdown",
license = "//lib:LICENSE-polymer",
deps = [
":iron-a11y-keys-behavior",
":iron-behaviors",
":iron-overlay-behavior",
":iron-resizable-behavior",
":neon-animation",
":polymer",
],
seed = True,
)
bower_component(
name = "iron-fit-behavior",
license = "//lib:LICENSE-polymer",
deps = [ ":polymer" ],
)
bower_component(
name = "iron-flex-layout",
license = "//lib:LICENSE-polymer",
deps = [ ":polymer" ],
)
bower_component(
name = "iron-form-element-behavior",
license = "//lib:LICENSE-polymer",
deps = [ ":polymer" ],
)
bower_component(
name = "iron-input",
license = "//lib:LICENSE-polymer",
deps = [
":iron-a11y-announcer",
":iron-validatable-behavior",
":polymer",
],
seed = True,
)
bower_component(
name = "iron-meta",
license = "//lib:LICENSE-polymer",
deps = [ ":polymer" ],
)
bower_component(
name = "iron-overlay-behavior",
license = "//lib:LICENSE-polymer",
deps = [
":iron-a11y-keys-behavior",
":iron-fit-behavior",
":iron-resizable-behavior",
":polymer",
],
seed = True,
)
bower_component(
name = "iron-resizable-behavior",
license = "//lib:LICENSE-polymer",
deps = [ ":polymer" ],
)
bower_component(
name = "iron-selector",
license = "//lib:LICENSE-polymer",
deps = [ ":polymer" ],
seed = True,
)
bower_component(
name = "iron-validatable-behavior",
license = "//lib:LICENSE-polymer",
deps = [
":iron-meta",
":polymer",
],
)
bower_component(
name = "moment",
license = "//lib:LICENSE-moment",
seed = True,
)
bower_component(
name = "neon-animation",
license = "//lib:LICENSE-polymer",
deps = [
":iron-meta",
":iron-resizable-behavior",
":iron-selector",
":polymer",
":web-animations-js",
],
)
bower_component(
name = "page",
license = "//lib:LICENSE-polymer",
seed = True,
)
bower_component(
name = "polymer",
license = "//lib:LICENSE-polymer",
deps = [ ":webcomponentsjs" ],
seed = True,
)
bower_component(
name = "promise-polyfill",
license = "//lib:LICENSE-polymer",
deps = [ ":polymer" ],
seed = True,
)
bower_component(
name = "web-animations-js",
license = "//lib:LICENSE-Apache2.0",
)
bower_component(
name = "webcomponentsjs",
license = "//lib:LICENSE-polymer",
)

34
polygerrit-ui/BUILD Normal file
View File

@ -0,0 +1,34 @@
load("//tools/bzl:js.bzl", "bower_component_bundle")
load('//tools/bzl:genrule2.bzl', 'genrule2')
bower_component_bundle(
name = "components",
deps = [
'//lib/js:es6-promise',
'//lib/js:fetch',
'//lib/js:highlightjs',
'//lib/js:iron-autogrow-textarea',
'//lib/js:iron-dropdown',
'//lib/js:iron-input',
'//lib/js:iron-overlay-behavior',
'//lib/js:iron-selector',
'//lib/js:moment',
'//lib/js:page',
'//lib/js:polymer',
'//lib/js:promise-polyfill',
])
genrule2(
name = 'fonts',
cmd = ' && '.join([
'cd $$TMP; for file in $(SRCS); do unzip -q $$ROOT/$$file; done',
'zip -q $$ROOT/$@ *',
]),
srcs = [
'//lib/fonts:sourcecodepro.zip',
],
out = 'fonts.zip',
visibility = ['//visibility:public'],
)

270
tools/bzl/js.bzl Normal file
View File

@ -0,0 +1,270 @@
NPMJS = "NPMJS"
GERRIT = "GERRIT"
NPM_VERSIONS = {
"bower": '1.7.9',
'crisper': '2.0.2',
'vulcanize': '1.14.8',
}
NPM_SHA1S = {
"bower": 'b7296c2393e0d75edaa6ca39648132dd255812b0',
"crisper": '7183c58cea33632fb036c91cefd1b43e390d22a2',
'vulcanize': '679107f251c19ab7539529b1e3fdd40829e6fc63',
}
def _npm_tarball(name):
return "%s@%s.npm_binary.tgz" % (name, NPM_VERSIONS[name])
def _npm_binary_impl(ctx):
"""rule to download a NPM archive."""
name = ctx.name
version= NPM_VERSIONS[name]
sha1 = NPM_VERSIONS[name]
dir = '%s-%s' % (name, version)
filename = '%s.tgz' % dir
base = '%s@%s.npm_binary.tgz' % (name, version)
dest = ctx.path(base)
repository = ctx.attr.repository
if repository == GERRIT:
url = 'http://gerrit-maven.storage.googleapis.com/npm-packages/%s' % filename
elif repository == NPMJS:
url = 'http://registry.npmjs.org/%s/-/%s' % (name, filename)
else:
fail('repository %s not in {%s,%s}' % (repository, GERRIT, NPMJS))
python = ctx.which("python")
script = ctx.path(ctx.attr._download_script)
args = [python, script, "-o", dest, "-u", url]
out = ctx.execute(args)
if out.return_code:
fail("failed %s: %s" % (args, out.stderr))
ctx.file("BUILD", "filegroup(name='tarball', srcs=['%s'])" % base, False)
npm_binary = repository_rule(
implementation=_npm_binary_impl,
local=True,
attrs={
# Label resolves within repo of the .bzl file.
"_download_script": attr.label(default=Label("//tools:download_file.py")),
"repository": attr.string(default=NPMJS),
})
def _run_npm_binary_str(ctx, tarball, name):
python_bin = ctx.which("python")
return " ".join([
python_bin,
ctx.path(ctx.attr._run_npm),
ctx.path(tarball)])
def _bower_archive(ctx):
"""Download a bower package."""
download_name = '%s__download_bower.zip' % ctx.name
renamed_name = '%s__renamed.zip' % ctx.name
version_name = '%s__version.json' % ctx.name
cmd = [
ctx.which("python"),
ctx.path(ctx.attr._download_bower),
'-b', '%s' % _run_npm_binary_str(ctx, ctx.attr._bower_archive, "bower"),
'-n', ctx.name,
'-p', ctx.attr.package,
'-v', ctx.attr.version,
'-s', ctx.attr.sha1,
'-o', download_name,
]
out = ctx.execute(cmd)
if out.return_code:
fail("failed %s: %s" % (" ".join(cmd), out.stderr))
_bash(ctx, " && " .join([
"TMP=$(mktemp -d )",
"cd $TMP",
"mkdir bower_components",
"cd bower_components",
"unzip %s" % ctx.path(download_name),
"cd ..",
"zip -r %s bower_components" % renamed_name,]))
dep_version = ctx.attr.semver if ctx.attr.semver else ctx.attr.version
ctx.file(version_name,
'"%s":"%s#%s"' % (ctx.name, ctx.attr.package, dep_version))
ctx.file(
"BUILD",
"\n".join([
"package(default_visibility=['//visibility:public'])",
"filegroup(name = 'zipfile', srcs = ['%s'], )" % download_name,
"filegroup(name = 'version_json', srcs = ['%s'], visibility=['//visibility:public'])" % version_name,
]), False)
def _bash(ctx, cmd):
cmd_list = ["/bin/bash", "-c", cmd]
out = ctx.execute(cmd_list)
if out.return_code:
fail("failed %s: %s", cmd_list, out.stderr)
bower_archive=repository_rule(
_bower_archive,
attrs={
"_bower_archive": attr.label(default=Label("@bower//:%s" % _npm_tarball("bower"))),
"_run_npm": attr.label(default=Label("//tools/js:run_npm_binary.py")),
"_download_bower": attr.label(default=Label("//tools/js:download_bower.py")),
"sha1": attr.string(mandatory=True),
"version": attr.string(mandatory=True),
"package": attr.string(mandatory=True),
"semver": attr.string(),
})
def _bower_component_impl(ctx):
transitive_zipfiles = set([ctx.file.zipfile])
for d in ctx.attr.deps:
transitive_zipfiles += d.transitive_zipfiles
transitive_licenses = set()
if ctx.file.license:
transitive_licenses += set([ctx.file.license])
for d in ctx.attr.deps:
transitive_licenses += d.transitive_licenses
transitive_versions = set(ctx.files.version_json)
for d in ctx.attr.deps:
transitive_versions += d.transitive_versions
return struct(
transitive_zipfiles=transitive_zipfiles,
transitive_versions=transitive_versions,
transitive_licenses=transitive_licenses,
)
_common_attrs = {
"deps": attr.label_list(providers=[
"transitive_zipfiles",
"transitive_versions",
"transitive_licenses",
])
}
def _js_component(ctx):
dir = ctx.outputs.zip.path + ".dir"
name = ctx.outputs.zip.basename
if name.endswith(".zip"):
name = name[:-4]
dest = "%s/%s" % (dir, name)
cmd = " && ".join([
"mkdir -p %s" % dest,
"cp %s %s/" % (' '.join([s.path for s in ctx.files.srcs]), dest),
"cd %s" % dir,
"find . -exec touch -t 198001010000 '{}' ';'",
"zip -qr ../%s *" % ctx.outputs.zip.basename
])
ctx.action(
inputs = ctx.files.srcs,
outputs = [ctx.outputs.zip],
command = cmd,
mnemonic = "GenBowerZip")
licenses = set()
if ctx.file.license:
licenses += set([ctx.file.license])
return struct(
transitive_zipfiles=list([ctx.outputs.zip]),
transitive_versions=set([]),
transitive_licenses=licenses)
js_component = rule(
_js_component,
attrs=_common_attrs + {
"srcs": attr.label_list(allow_files=[".js"]),
"license": attr.label(allow_single_file=True),
},
outputs={
"zip": "%{name}.zip",
}
)
_bower_component = rule(
_bower_component_impl,
attrs=_common_attrs + {
"zipfile": attr.label(allow_single_file=[".zip"]),
"license": attr.label(allow_single_file=True),
"version_json": attr.label(allow_files=[".json"]),
# If set, define by hand, and don't regenerate this entry in bower2bazel.
"seed": attr.bool(default=False)
})
# TODO(hanwen): make license mandatory.
def bower_component(name, license=None, **kwargs):
prefix = "//lib:LICENSE-"
if license and not license.startswith(prefix):
license = prefix + license
_bower_component(
name=name,
license=license,
zipfile="@%s//:zipfile"% name,
version_json="@%s//:version_json" % name,
**kwargs)
def _bower_component_bundle_impl(ctx):
"""A bunch of bower components zipped up."""
zips = set([])
for d in ctx.attr.deps:
zips += d.transitive_zipfiles
versions = set([])
for d in ctx.attr.deps:
versions += d.transitive_versions
out_zip = ctx.outputs.zip
out_versions = ctx.outputs.version_json
ctx.action(
inputs=list(zips),
outputs=[out_zip],
command=" && ".join([
"p=$PWD",
"rm -rf %s.dir" % out_zip.path,
"mkdir -p %s.dir/bower_components" % out_zip.path,
"cd %s.dir/bower_components" % out_zip.path,
"for z in %s; do unzip -q $p/$z ; done" % " ".join(sorted([z.path for z in zips])),
"cd ..",
"find . -exec touch -t 198001010000 '{}' ';'",
"zip -qr $p/%s bower_components/*" % out_zip.path,
]),
mnemonic="BowerCombine")
ctx.action(
inputs=list(versions),
outputs=[out_versions],
mnemonic="BowerVersions",
command="(echo '{' ; for j in %s ; do cat $j; echo ',' ; done ; echo \\\"\\\":\\\"\\\"; echo '}') > %s" % (" ".join([v.path for v in versions]), out_versions.path))
bower_component_bundle = rule(
_bower_component_bundle_impl,
attrs=_common_attrs,
outputs={
"zip": "%{name}.zip",
"version_json": "%{name}-versions.json",
}
)

View File

@ -84,7 +84,7 @@ opts.add_option('--unsign', action='store_true')
args, _ = opts.parse_args()
root_dir = args.o
while root_dir:
while root_dir and root_dir != "/":
root_dir, n = path.split(root_dir)
if n == 'buck-out':
break

View File

@ -1,14 +1,26 @@
python_binary(
name = 'bower2buck',
main = 'bower2buck.py',
deps = ['//tools:util'],
deps = [
'//tools:util',
":bowerutil",
],
visibility = ['PUBLIC'],
)
python_library(
name = 'bowerutil',
srcs = [ 'bowerutil.py' ],
visibility = [ 'PUBLIC' ],
)
python_binary(
name = 'download_bower',
main = 'download_bower.py',
deps = ['//tools:util'],
deps = [
'//tools:util',
":bowerutil",
],
visibility = ['PUBLIC'],
)

0
tools/js/BUILD Normal file
View File

228
tools/js/bower2bazel.py Executable file
View File

@ -0,0 +1,228 @@
#!/usr/bin/env python
# Copyright (C) 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Suggested call sequence:
python tools/js/bower2bazel.py -w lib/js/bower_archives.bzl -b lib/js/bower_components.bzl
"""
from __future__ import print_function
import atexit
import collections
import json
import hashlib
import optparse
import os
import shutil
import subprocess
import sys
import tempfile
import glob
import bowerutil
# Map license names to our canonical names.
license_map = {
"http://polymer.github.io/LICENSE.txt": "polymer",
"Apache-2.0": "Apache2.0",
# TODO(hanwen): remove these, and add appropriate license files under //lib
"BSD": "polymer",
"MIT": "polymer",
}
# list of licenses for packages that don't specify one in their bower.json file.
package_licenses = {
"es6-promise": "es6-promise",
"fetch": "fetch",
"moment": "moment",
"page": "page.js",
"promise-polyfill": "promise-polyfill",
"webcomponentsjs": "polymer", # self-identifies as BSD.
}
def build_bower_json(version_targets):
"""Generate bower JSON file, return its path."""
bower_json = collections.OrderedDict()
bower_json['name'] = 'bower2buck-output'
bower_json['version'] = '0.0.0'
bower_json['description'] = 'Auto-generated bower.json for dependency management'
bower_json['private'] = True
bower_json['dependencies'] = {}
for v in version_targets:
fn = os.path.join("bazel-out/local-fastbuild/bin", v.lstrip("/").replace(":", "/"))
with open(fn) as f:
j = json.load(f)
if "" in j:
# drop dummy entries.
del j[""]
bower_json['dependencies'].update(j)
tmpdir = tempfile.mkdtemp()
ret = os.path.join(tmpdir, 'bower.json')
with open(ret, 'w') as f:
json.dump(bower_json, f, indent=2)
return ret
def bower_command(args):
base = subprocess.check_output(["bazel", "info", "output_base"]).strip()
exp = os.path.join(base, "external", "bower", "*npm_binary.tgz")
fs = sorted(glob.glob(exp))
assert len(fs) == 1, "bower tarball not found or have multiple versions %s" % fs
return ["python", os.getcwd() + "/tools/js/run_npm_binary.py", sorted(fs)[0]] + args
def main(args):
opts = optparse.OptionParser()
opts.add_option('-w', help='.bzl output for WORKSPACE')
opts.add_option('-b', help='.bzl output for //lib:BUILD')
opts, args = opts.parse_args()
target_str = subprocess.check_output([
"bazel", "query", "kind(bower_component_bundle, //polygerrit-ui/...)"])
seed_str = subprocess.check_output([
"bazel", "query", "attr(seed, 1, kind(bower_component, deps(//polygerrit-ui/...)))"])
targets = [s for s in target_str.split('\n') if s]
seeds = [s for s in seed_str.split('\n') if s]
prefix = "//lib/js:"
non_seeds = [s for s in seeds if not s.startswith(prefix)]
assert not non_seeds, non_seeds
seeds = set([s[len(prefix):] for s in seeds])
version_targets = [t + "-versions.json" for t in targets]
subprocess.check_call(['bazel', 'build'] + version_targets)
bower_json_path = build_bower_json(version_targets)
dir = os.path.dirname(bower_json_path)
cmd = bower_command(["install"])
build_out = sys.stdout
if opts.b:
build_out = open(opts.b + ".tmp", 'w')
ws_out = sys.stdout
if opts.b:
ws_out = open(opts.w + ".tmp", 'w')
header = """# DO NOT EDIT
# generated with the following command:
#
# %s
#
""" % ' '.join(sys.argv)
ws_out.write(header)
build_out.write(header)
oldwd = os.getcwd()
os.chdir(dir)
subprocess.check_call(cmd)
interpret_bower_json(seeds, ws_out, build_out)
ws_out.close()
build_out.close()
os.chdir(oldwd)
os.rename(opts.w + ".tmp", opts.w)
os.rename(opts.b + ".tmp", opts.b)
def dump_workspace(data, seeds, out):
out.write('load("//tools/bzl:js.bzl", "bower_archive")\n')
out.write('def load_bower_archives():\n')
for d in data:
if d["name"] in seeds:
continue
out.write(""" bower_archive(
name = "%(name)s",
package = "%(normalized-name)s",
version = "%(version)s",
sha1 = "%(bazel-sha1)s")
""" % d)
def dump_build(data, seeds, out):
out.write('load("//tools/bzl:js.bzl", "bower_component")\n')
out.write('def define_bower_components():\n')
for d in data:
out.write(" bower_component(\n")
out.write(" name = \"%s\",\n" % d["name"])
out.write(" license = \"//lib:LICENSE-%s\",\n" % d["bazel-license"])
deps = sorted(d.get("dependencies", {}).keys())
if deps:
if len(deps) == 1:
out.write(" deps = [ \":%s\" ],\n" % deps[0])
else:
out.write(" deps = [\n")
for dep in deps:
out.write(" \":%s\",\n" % dep)
out.write(" ],\n")
if d["name"] in seeds:
out.write(" seed = True,\n")
out.write(" )\n")
# done
def interpret_bower_json(seeds, ws_out, build_out):
out = subprocess.check_output(["find", "bower_components/", "-name", ".bower.json"])
data = []
for f in sorted(out.split('\n')):
if not f:
continue
pkg = json.load(open(f))
pkg_name = pkg["name"]
pkg["bazel-sha1"] = bowerutil.hash_bower_component(
hashlib.sha1(), os.path.dirname(f)).hexdigest()
license = pkg.get("license", None)
if type(license) == type([]):
# WTF? Some package specify a list of licenses. ("GPL", "MIT")
pick = license[0]
sys.stderr.write("package %s has multiple licenses: %s, picking %s" % (pkg_name, ", ".join(license), pick))
license = pick
if license:
license = license_map.get(license, license)
else:
if pkg_name not in package_licenses:
msg = "package %s does not specify license." % pkg_name
sys.stderr.write(msg)
raise Exception(msg)
license = package_licenses[pkg_name]
pkg["bazel-license"] = license
# TODO(hanwen): bower packages can also have 'fully qualified'
# names, ("PolymerElements/iron-ajax") as well as short names
# ("iron-ajax"). It is possible for bower.json files to refer to
# long names as their dependencies. If any package does this, we
# will have to either 1) strip off the prefix (typically github
# user?), or 2) build a map of short name <=> fully qualified
# name. For now, we just ignore the problem.
pkg["normalized-name"] = pkg["name"]
data.append(pkg)
dump_workspace(data, seeds, ws_out)
dump_build(data, seeds, build_out)
if __name__ == '__main__':
main(sys.argv[1:])

View File

@ -78,7 +78,7 @@ class Rule(object):
self.version = bower_json['version']
self.deps = bower_json.get('dependencies', {})
self.license = bower_json.get('license', 'NO LICENSE')
self.sha1 = util.hash_bower_component(
self.sha1 = bowerutil.hash_bower_component(
hashlib.sha1(), os.path.dirname(bower_json_path)).hexdigest()
def to_rule(self, packages):
@ -106,6 +106,7 @@ class Rule(object):
def build_bower_json(targets, buck_out):
"""create bower.json so 'bower install' fetches transitive deps"""
bower_json = collections.OrderedDict()
bower_json['name'] = 'bower2buck-output'
bower_json['version'] = '0.0.0'
@ -117,6 +118,9 @@ def build_bower_json(targets, buck_out):
['buck', 'query', '-v', '0',
"filter('__download_bower', deps(%s))" % '+'.join(targets)],
env=BUCK_ENV)
# __bower_version contains the version number coming from version
# attr in BUCK/BUILD
deps = deps.replace('__download_bower', '__bower_version').split()
subprocess.check_call(['buck', 'build'] + deps, env=BUCK_ENV)

47
tools/js/bowerutil.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright (C) 2013 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from os import path
def hash_bower_component(hash_obj, path):
"""Hash the contents of a bower component directory.
This is a stable hash of a directory downloaded with `bower install`, minus
the .bower.json file, which is autogenerated each time by bower. Used in lieu
of hashing a zipfile of the contents, since zipfiles are difficult to hash in
a stable manner.
Args:
hash_obj: an open hash object, e.g. hashlib.sha1().
path: path to the directory to hash.
Returns:
The passed-in hash_obj.
"""
if not os.path.isdir(path):
raise ValueError('Not a directory: %s' % path)
path = os.path.abspath(path)
for root, dirs, files in os.walk(path):
dirs.sort()
for f in sorted(files):
if f == '.bower.json':
continue
p = os.path.join(root, f)
hash_obj.update(p[len(path)+1:])
hash_obj.update(open(p).read())
return hash_obj

21
tools/js/download_bower.py Normal file → Executable file
View File

@ -23,8 +23,7 @@ import shutil
import subprocess
import sys
from tools import util
import bowerutil
CACHE_DIR = os.path.expanduser(os.path.join(
'~', '.gerritcodereview', 'buck-cache', 'downloaded-artifacts'))
@ -39,16 +38,20 @@ def bower_cmd(bower, *args):
def bower_info(bower, name, package, version):
cmd = bower_cmd(bower, '-l=error', '-j',
'info', '%s#%s' % (package, version))
p = subprocess.Popen(cmd , stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
p = subprocess.Popen(cmd , stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except:
sys.stderr.write("error executing: %s\n" % ' '.join(cmd))
raise
out, err = p.communicate()
if p.returncode:
sys.stderr.write(err)
raise OSError('Command failed: %s' % cmd)
raise OSError('Command failed: %s' % ' '.join(cmd))
try:
info = json.loads(out)
except ValueError:
raise ValueError('invalid JSON from %s:\n%s' % (cmd, out))
raise ValueError('invalid JSON from %s:\n%s' % (" ".join(cmd), out))
info_name = info.get('name')
if info_name != name:
raise ValueError('expected package name %s, got: %s' % (name, info_name))
@ -82,7 +85,11 @@ def main(args):
opts.add_option('-v', help='version number')
opts.add_option('-s', help='expected content sha1')
opts.add_option('-o', help='output file location')
opts, _ = opts.parse_args()
opts, args_ = opts.parse_args(args)
assert opts.p
assert opts.v
assert opts.n
cwd = os.getcwd()
outzip = os.path.join(cwd, opts.o)
@ -100,7 +107,7 @@ def main(args):
if opts.s:
path = os.path.join(bc, opts.n)
sha1 = util.hash_bower_component(hashlib.sha1(), path).hexdigest()
sha1 = bowerutil.hash_bower_component(hashlib.sha1(), path).hexdigest()
if opts.s != sha1:
print((
'%s#%s:\n'

View File

@ -25,8 +25,6 @@ import sys
import tarfile
import tempfile
from tools import util
def extract(path, outdir, bin):
if os.path.exists(os.path.join(outdir, bin)):
@ -59,19 +57,21 @@ def extract(path, outdir, bin):
# finished.
extract_one(tar.getmember(bin))
def main(args):
path = args[0]
suffix = '.npm_binary.tgz'
tgz = os.path.basename(path)
parts = tgz[:-len(suffix)].split('@')
if not tgz.endswith(suffix) or len(parts) != 2:
print('usage: %s <path/to/npm_binary>' % sys.argv[0], file=sys.stderr)
return 1
name, version = parts
sha1 = util.hash_file(hashlib.sha1(), path).hexdigest()
name, _ = parts
# Avoid importing from gerrit because we don't want to depend on the right CWD.
sha1 = hashlib.sha1(open(path, 'rb').read()).hexdigest()
outdir = '%s-%s' % (path[:-len(suffix)], sha1)
rel_bin = os.path.join('package', 'bin', name)
bin = os.path.join(outdir, rel_bin)

View File

@ -70,34 +70,3 @@ def hash_file(hash_obj, path):
break
hash_obj.update(b)
return hash_obj
def hash_bower_component(hash_obj, path):
"""Hash the contents of a bower component directory.
This is a stable hash of a directory downloaded with `bower install`, minus
the .bower.json file, which is autogenerated each time by bower. Used in lieu
of hashing a zipfile of the contents, since zipfiles are difficult to hash in
a stable manner.
Args:
hash_obj: an open hash object, e.g. hashlib.sha1().
path: path to the directory to hash.
Returns:
The passed-in hash_obj.
"""
if not os.path.isdir(path):
raise ValueError('Not a directory: %s' % path)
path = os.path.abspath(path)
for root, dirs, files in os.walk(path):
dirs.sort()
for f in sorted(files):
if f == '.bower.json':
continue
p = os.path.join(root, f)
hash_obj.update(p[len(path)+1:])
hash_file(hash_obj, p)
return hash_obj