Support downloading from other Maven repositories

Developers may now request buck to use a local mirror by setting
the URL into local.properties, a file already ignored under the
current buck build process, e.g.:

  download.GERRIT = http://nexus.my-company.com/
  download.MAVEN_CENTRAL = http://nexus.my-company.com/

To support this usage buck now passes to download_jar.py only the name
of the repository in the URL.  download_jar parses local.properties
(if present) and replaces the "MAVEN_CENTRAL:" prefix with the URL
supplied by the user.

Because the URL can vary cache entries in buck-cache are now using the
SHA-1 we expect/verify against, instead of the SHA-1 of the URL.  This
makes it easier to find a specific JAR in the cache.  The artifact,
version and SHA-1 as named in the maven_jar() rule are the strings
used in the file name in buck-cache.

If no SHA-1 verification happens (e.g.  source attachment JARs that
are not included into the build) then sha1(canonical_url) is used.
Here the canonical URL uses the repository id prefix string, e.g.:

  MAVEN_CENTRAL:org/apache/mina/mina-core/2.0.5/mina-core-2.0.5-sources.jar

Change-Id: I5a469ab15fd36b81bf76f6b51e1110dfdcf04c86

Documentation/dev-buck.txt

@@ -136,11 +136,6 @@ Force a download of dependency JARs by building the `download` target:
   buck build download
 ----
 
-Currently there is no way to download dependencies other than over an HTTP
-connection from the source sites (Maven Central and the Gerrit storage
-bucket).  It is not yet possible to use, for example, a corporate Nexus
-proxy.
-
 When downloading from behind a proxy (which is common in some corporate
 environments), it might be necessary to explicitly specify the proxy that
 is then used by `curl`:
@@ -149,6 +144,15 @@ is then used by `curl`:
   export http_proxy=http://<proxy_user_id>:<proxy_password>@<proxy_server>:<proxy_port>
 ----
 
+Redirection to local mirrors of Maven Central and the Gerrit storage
+bucket is supported by defining specific properties in
+`local.properties`, a file that is not tracked by Git:
+
+----
+  echo download.GERRIT = http://nexus.my-company.com/ >>local.properties
+  echo download.MAVEN_CENTRAL = http://nexus.my-company.com/ >>local.properties
+----
+
 
 Build Process Switch Exit Criteria
 ----------------------------------

lib/maven.defs

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-GERRIT = 'http://gerrit-maven.commondatastorage.googleapis.com'
+GERRIT = 'GERRIT:'
-MAVEN_CENTRAL = 'http://repo1.maven.org/maven2'
+MAVEN_CENTRAL = 'MAVEN_CENTRAL:'
 
 def define_license(name):
   n = 'LICENSE-' + name

tools/download_jar.py

@@ -23,6 +23,11 @@ from subprocess import check_call, CalledProcessError
 from sys import stderr
 from zipfile import ZipFile, BadZipfile, LargeZipFile
 
+REPO_ROOTS = {
+  'GERRIT': 'http://gerrit-maven.commondatastorage.googleapis.com',
+  'MAVEN_CENTRAL': 'http://repo1.maven.org/maven2',
+}
+
 def hashfile(p):
   d = sha1()
   with open(p, 'rb') as f:
@@ -42,6 +47,46 @@ def safe_mkdirs(d):
     if not path.isdir(d):
       raise err
 
+def download_properties(root_dir):
+  local_prop = path.join(root_dir, 'local.properties')
+  p = {}
+  if path.isfile(local_prop):
+    try:
+      with open(local_prop) as fd:
+        for line in fd:
+          if line.startswith('download.'):
+            d = [e.strip() for e in line.split('=', 1)]
+            name, url = d[0], d[1]
+            p[name[len('download.'):]] = url
+    except OSError as err:
+      pass
+  return p
+
+def cache_entry(root_dir, args):
+  if args.v:
+    h = args.v
+  else:
+    h = sha1(args.u).hexdigest()
+  name = '%s-%s' % (path.basename(args.o), h)
+  return path.join(root_dir, 'buck-cache', name)
+
+def resolve_url(url, redirects):
+  s = url.find(':')
+  if s < 0:
+    return url
+  scheme, rest = url[:s], url[s+1:]
+  if scheme not in REPO_ROOTS:
+    return url
+  if scheme in redirects:
+    root = redirects[scheme]
+  else:
+    root = REPO_ROOTS[scheme]
+  while root.endswith('/'):
+    root = root[:-1]
+  while rest.startswith('/'):
+    rest = rest[1:]
+  return '/'.join([root, rest])
+
 opts = OptionParser()
 opts.add_option('-o', help='local output file')
 opts.add_option('-u', help='URL to download')
@@ -56,16 +101,15 @@ while root_dir:
   if n == 'buck-out':
     break
 
-cache_ent = path.join(
+redirects = download_properties(root_dir)
-    root_dir,
+cache_ent = cache_entry(root_dir, args)
-    'buck-cache',
+src_url = resolve_url(args.u, redirects)
-    '%s-%s' % (path.basename(args.o), sha1(args.u).hexdigest()))
 
 if not path.exists(cache_ent):
   try:
     safe_mkdirs(path.dirname(cache_ent))
-    print('Download %s' % args.u, file=stderr)
+    print('Download %s' % src_url, file=stderr)
-    check_call(['curl', '--proxy-anyauth', '-sfo', cache_ent, args.u])
+    check_call(['curl', '--proxy-anyauth', '-sfo', cache_ent, src_url])
   except (OSError, CalledProcessError) as err:
     print('error using curl: %s' % str(err), file=stderr)
     exit(1)
@@ -78,7 +122,7 @@ if args.v:
       '%s:\n' +
       'expected %s\n' +
       'received %s\n' +
-      '         %s\n') % (args.u, args.v, have, o), file=stderr)
+      '         %s\n') % (src_url, args.v, have, o), file=stderr)
     exit(1)
 
 exclude = []