Buck: Support plugin own Maven repositories

Currently only predefined Maven repositories are supported by Buck maven_jar.
Additional logic exists to redirect to a local repository mirror.

Current implementation relies on the repository name matching between repo
passed to maven_jar and redirect definition defined in local.properties that
is not under Git control.

This change extends that by allowing to pass not only the repo name but the
complete URL to maven_jar. The augmented implementation checks if it is a
known Maven repository: if it is, then the behavious is unchanged, if not,
then the passed URL is used.

As result plugin's can use custom Maven repositories:

  GERRIT_FORGE = 'http://gerritforge.com/snapshot'

  maven_jar(
    name = 'gitblit',
    id = 'com.gitblit:gitblit:1.4.0',
    sha1 = '1b130dbf5578ace37507430a4a523f6594bf34fa',
    license = 'Apache2.0',
    repository = GERRIT_FORGE,
  )

Plugin owned Maven repositories can also be rewritten in local.properties.
To achieve that custom repository name must be passed to the maven_jar()
function, like known repositories, and the URL must be defined in
local.properties.

local.properties excerpt:

  download.GERRIT_FORGE = http://my.company.mirror/gerrit-forge

BUCK excerpt:

  GERRIT_FORGE = 'GERRIT_FORGE:'

  maven_jar(
    name = 'gitblit',
    id = 'com.gitblit:gitblit:1.4.0',
    sha1 = '1b130dbf5578ace37507430a4a523f6594bf34fa',
    license = 'Apache2.0',
    repository = GERRIT_FORGE,
  )

Python unit test can be executed with other Java unit tests:

  buck test tools:python_tests
Change-Id: Ib31d51f0884b1ca1a07b6492f861f404db115946

Documentation/dev-buck.txt

@@ -317,6 +317,51 @@ that artifact:
  )
 ----
 
+Building against artifacts from custom Maven repositories
+---------------------------------------------------------
+
+To build against custom Maven repositories, two modes of operations are
+supported: with rewrite in local.properties and without.
+
+Without rewrite the URL of custom Maven repository can be directly passed
+to the maven_jar() function:
+
+[source,python]
+----
+  GERRIT_FORGE = 'http://gerritforge.com/snapshot'
+
+  maven_jar(
+    name = 'gitblit',
+    id = 'com.gitblit:gitblit:1.4.0',
+    sha1 = '1b130dbf5578ace37507430a4a523f6594bf34fa',
+    license = 'Apache2.0',
+    repository = GERRIT_FORGE,
+ )
+----
+
+When the custom URL has to be rewritten, then the same logic as with Gerrit
+known Maven repository is used: Repo name must be defined that matches an entry
+in local.properties file:
+
+----
+  download.GERRIT_FORGE = http://my.company.mirror/gerrit-forge
+----
+
+And corresponding BUCK excerpt:
+
+[source,python]
+----
+  GERRIT_FORGE = 'GERRIT_FORGE:'
+
+  maven_jar(
+    name = 'gitblit',
+    id = 'com.gitblit:gitblit:1.4.0',
+    sha1 = '1b130dbf5578ace37507430a4a523f6594bf34fa',
+    license = 'Apache2.0',
+    repository = GERRIT_FORGE,
+ )
+----
+
 Caching Build Results
 ~~~~~~~~~~~~~~~~~~~~~
 

tools/BUCK

@@ -1,6 +1,7 @@
 python_binary(
   name = 'download_file',
   main = 'download_file.py',
+  deps = [':util'],
   visibility = ['PUBLIC'],
 )
 
@@ -17,6 +18,13 @@ python_library(
   visibility = ['PUBLIC'],
 )
 
+python_library(
+  name = 'util_test',
+  srcs = ['util_test.py'],
+  deps = [':util'],
+  visibility = ['PUBLIC'],
+)
+
 def shquote(s):
   return s.replace("'", "'\\''")
 
@@ -32,3 +40,14 @@ genrule(
   out = 'buck.properties',
   visibility = ['PUBLIC'],
 )
+
+java_test(
+  name = 'python_tests',
+  srcs = glob(['PythonTestCaller.java']),
+  deps = [
+    '//lib:guava',
+    '//lib:junit',
+    ':util',
+    ':util_test',
+  ],
+)

tools/PythonTestCaller.java

@@ -0,0 +1,67 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.io.ByteStreams;
+import com.google.common.base.Splitter;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import org.junit.Test;
+
+public class PythonTestCaller {
+
+  @Test
+  public void resolveUrl() throws Exception {
+    PythonTestCaller.pythonUnit("tools", "util_test");
+  }
+
+  private static void pythonUnit(String d, String sut) throws Exception {
+    ProcessBuilder b =
+        new ProcessBuilder(Splitter.on(' ').splitToList(
+                "python -m unittest " + sut))
+            .directory(new File(d))
+            .redirectErrorStream(true);
+    Process p = null;
+    InputStream i = null;
+    byte[] out;
+    try {
+      p = b.start();
+      i = p.getInputStream();
+      out = ByteStreams.toByteArray(i);
+    } catch (IOException e) {
+      throw new Exception(e);
+    } finally {
+      if (p != null) {
+        p.getOutputStream().close();
+      }
+      if (i != null) {
+        i.close();
+      }
+    }
+    int value;
+    try {
+      value = p.waitFor();
+    } catch (InterruptedException e) {
+      throw new Exception("interrupted waiting for process");
+    }
+    String err = new String(out, "UTF-8");
+    if (value != 0) {
+      System.err.print(err);
+    }
+    assertTrue(err, value == 0);
+  }
+}

tools/download_file.py

@@ -21,20 +21,13 @@ from os import link, makedirs, path, remove
 import shutil
 from subprocess import check_call, CalledProcessError
 from sys import stderr
+from util import resolve_url
 from zipfile import ZipFile, BadZipfile, LargeZipFile
 
-REPO_ROOTS = {
-  'GERRIT': 'http://gerrit-maven.storage.googleapis.com',
-  'ECLIPSE': 'https://repo.eclipse.org/content/groups/releases',
-  'MAVEN_CENTRAL': 'http://repo1.maven.org/maven2',
-  'MAVEN_LOCAL': 'file://' + path.expanduser('~/.m2/repository'),
-}
-
 GERRIT_HOME = path.expanduser('~/.gerritcodereview')
 CACHE_DIR = path.join(GERRIT_HOME, 'buck-cache')
 LOCAL_PROPERTIES = 'local.properties'
 
-
 def hashfile(p):
   d = sha1()
   with open(p, 'rb') as f:
@@ -88,21 +81,6 @@ def cache_entry(args):
   name = '%s-%s' % (path.basename(args.o), h)
   return path.join(CACHE_DIR, name)
 
-def resolve_url(url, redirects):
-  s = url.find(':')
-  if s < 0:
-    return url
-  scheme, rest = url[:s], url[s+1:]
-  if scheme not in REPO_ROOTS:
-    return url
-  if scheme in redirects:
-    root = redirects[scheme]
-  else:
-    root = REPO_ROOTS[scheme]
-  root = root.rstrip('/')
-  rest = rest.lstrip('/')
-  return '/'.join([root, rest])
-
 opts = OptionParser()
 opts.add_option('-o', help='local output file')
 opts.add_option('-u', help='URL to download')

tools/util.py

@@ -12,9 +12,46 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from os import path
+
 try:
   from subprocess import check_output
 except ImportError:
   from subprocess import Popen, PIPE
   def check_output(*cmd):
     return Popen(*cmd, stdout=PIPE).communicate()[0]
+
+REPO_ROOTS = {
+  'GERRIT': 'http://gerrit-maven.storage.googleapis.com',
+  'ECLIPSE': 'https://repo.eclipse.org/content/groups/releases',
+  'MAVEN_CENTRAL': 'http://repo1.maven.org/maven2',
+  'MAVEN_LOCAL': 'file://' + path.expanduser('~/.m2/repository'),
+}
+
+def resolve_url(url, redirects):
+  """ Resolve URL of a Maven artifact.
+
+  prefix:path is passed as URL. prefix identifies known or custom
+  repositories that can be rewritten in redirects set, passed as
+  second arguments.
+
+  A special case is supported, when prefix neither exists in
+  REPO_ROOTS, no in redirects set: the url is returned as is.
+  This enables plugins to pass custom maven_repository URL as is
+  directly to maven_jar().
+
+  Returns a resolved path for Maven artifact.
+  """
+  s = url.find(':')
+  if s < 0:
+    return url
+  scheme, rest = url[:s], url[s+1:]
+  if scheme in redirects:
+    root = redirects[scheme]
+  elif scheme in REPO_ROOTS:
+    root = REPO_ROOTS[scheme]
+  else:
+    return url
+  root = root.rstrip('/')
+  rest = rest.lstrip('/')
+  return '/'.join([root, rest])

tools/util_test.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from util import resolve_url
+
+class TestResolveUrl(unittest.TestCase):
+  """ run to test:
+    python -m unittest -v util_test
+  """
+
+  def testKnown(self):
+    url = resolve_url('GERRIT:foo.jar', {})
+    self.assertEqual(url, 'http://gerrit-maven.storage.googleapis.com/foo.jar')
+
+  def testKnownRedirect(self):
+    url = resolve_url('MAVEN_CENTRAL:foo.jar',
+                      {'MAVEN_CENTRAL': 'http://my.company.mirror/maven2'})
+    self.assertEqual(url, 'http://my.company.mirror/maven2/foo.jar')
+
+  def testCustom(self):
+    url = resolve_url('http://maven.example.com/release/foo.jar', {})
+    self.assertEqual(url, 'http://maven.example.com/release/foo.jar')
+
+  def testCustomRedirect(self):
+    url = resolve_url('MAVEN_EXAMPLE:foo.jar',
+                      {'MAVEN_EXAMPLE': 'http://maven.example.com/release'})
+    self.assertEqual(url, 'http://maven.example.com/release/foo.jar')
+
+if __name__ == '__main__':
+  unittest.main()