Add support for media uploads as they will appear in Discovery.

Reviewed in http://codereview.appspot.com/4664050/

apiclient/discovery.py

@@ -29,6 +29,7 @@ import re
 import uritemplate
 import urllib
 import urlparse
+import mimeparse
 import mimetypes
 
 try:
@@ -41,6 +42,8 @@ from email.mime.multipart import MIMEMultipart
 from email.mime.nonmultipart import MIMENonMultipart
 from errors import HttpError
 from errors import InvalidJsonError
+from errors import MediaUploadSizeError
+from errors import UnacceptableMimeTypeError
 from errors import UnknownLinkType
 from http import HttpRequest
 from model import JsonModel
@@ -226,6 +229,22 @@ def _cast(value, schema_type):
     else:
       return str(value)
 
+MULTIPLIERS = {
+    "KB": 2**10,
+    "MB": 2**20,
+    "GB": 2**30,
+    "TB": 2**40,
+    }
+
+def _media_size_to_long(maxSize):
+  """Convert a string media size, such as 10GB or 3TB into an integer."""
+  units = maxSize[-2:].upper()
+  multiplier = MULTIPLIERS.get(units, 0)
+  if multiplier:
+    return int(maxSize[:-2])*multiplier
+  else:
+    return int(maxSize)
+
 
 def createResource(http, baseUrl, model, requestBuilder,
                    developerKey, resourceDesc, futureDesc):
@@ -245,6 +264,15 @@ def createResource(http, baseUrl, model, requestBuilder,
     httpMethod = methodDesc['httpMethod']
     methodId = methodDesc['id']
 
+    mediaPathUrl = None
+    accept = []
+    maxSize = 0
+    if 'mediaUpload' in methodDesc:
+      mediaUpload = methodDesc['mediaUpload']
+      mediaPathUrl = mediaUpload['protocols']['simple']['path']
+      accept = mediaUpload['accept']
+      maxSize = _media_size_to_long(mediaUpload['maxSize'])
+
     if 'parameters' not in methodDesc:
       methodDesc['parameters'] = {}
     for name in STACK_QUERY_PARAMETERS:
@@ -259,11 +287,13 @@ def createResource(http, baseUrl, model, requestBuilder,
           'type': 'object',
           'required': True,
           }
-      methodDesc['parameters']['media_body'] = {
-          'description': 'The filename of the media request body.',
-          'type': 'string',
-          'required': False,
-          }
+      if 'mediaUpload' in methodDesc:
+        methodDesc['parameters']['media_body'] = {
+            'description': 'The filename of the media request body.',
+            'type': 'string',
+            'required': False,
+            }
+        methodDesc['parameters']['body']['required'] = False
 
     argmap = {} # Map from method parameter name to query parameter name
     required_params = [] # Required parameters
@@ -324,7 +354,6 @@ def createResource(http, baseUrl, model, requestBuilder,
                 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
                 (name, kwargs[name], str(enums)))
 
-      media_filename = kwargs.pop('media_body', None)
       actual_query_params = {}
       actual_path_params = {}
       for key, value in kwargs.iteritems():
@@ -339,6 +368,7 @@ def createResource(http, baseUrl, model, requestBuilder,
         if key in path_params:
           actual_path_params[argmap[key]] = cast_value
       body_value = kwargs.get('body', None)
+      media_filename = kwargs.get('media_body', None)
 
       if self._developerKey:
         actual_query_params['key'] = self._developerKey
@@ -354,11 +384,16 @@ def createResource(http, baseUrl, model, requestBuilder,
         (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
         if media_mime_type is None:
           raise UnknownFileType(media_filename)
+        if not mimeparse.best_match([media_mime_type], ','.join(accept)):
+          raise UnacceptableMimeTypeError(media_mime_type)
 
-        # modify the path to prepend '/upload'
-        parsed = list(urlparse.urlparse(url))
-        parsed[2] = '/upload' + parsed[2]
-        url = urlparse.urlunparse(parsed)
+        # Check the maxSize
+        if maxSize > 0 and os.path.getsize(media_filename) > maxSize:
+          raise MediaUploadSizeError(media_filename)
+
+        # Use the media path uri for media uploads
+        expanded_url = uritemplate.expand(mediaPathUrl, params)
+        url = urlparse.urljoin(self._baseUrl, expanded_url + query)
 
         if body is None:
           headers['content-type'] = media_mime_type
@@ -406,8 +441,6 @@ def createResource(http, baseUrl, model, requestBuilder,
     for arg in argmap.iterkeys():
       if arg in STACK_QUERY_PARAMETERS:
         continue
-      if arg == 'media_body':
-        continue
       repeated = ''
       if arg in repeated_params:
         repeated = ' (repeated)'

apiclient/errors.py

@@ -40,8 +40,7 @@ class HttpError(Error):
     self.uri = uri
 
   def _get_reason(self):
-    """Calculate the reason for the error from the response content.
-    """
+    """Calculate the reason for the error from the response content."""
     if self.resp.get('content-type', '').startswith('application/json'):
       try:
         data = simplejson.loads(self.content)
@@ -70,3 +69,11 @@ class InvalidJsonError(Error):
 class UnknownLinkType(Error):
   """Link type unknown or unexpected."""
   pass
+
+class UnacceptableMimeTypeError(Error):
+  """That is an unacceptable mimetype for this operation."""
+  pass
+
+class MediaUploadSizeError(Error):
+  """Media is larger than the method can accept."""
+  pass

apiclient/mimeparse.py

@@ -0,0 +1,172 @@
+# Copyright (C) 2007 Joe Gregorio
+#
+# Licensed under the MIT License
+
+"""MIME-Type Parser
+
+This module provides basic functions for handling mime-types. It can handle
+matching mime-types against a list of media-ranges. See section 14.1 of the
+HTTP specification [RFC 2616] for a complete explanation.
+
+   http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1
+
+Contents:
+ - parse_mime_type():   Parses a mime-type into its component parts.
+ - parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q'
+                          quality parameter.
+ - quality():           Determines the quality ('q') of a mime-type when
+                          compared against a list of media-ranges.
+ - quality_parsed():    Just like quality() except the second parameter must be
+                          pre-parsed.
+ - best_match():        Choose the mime-type with the highest quality ('q')
+                          from a list of candidates.
+"""
+
+__version__ = '0.1.3'
+__author__ = 'Joe Gregorio'
+__email__ = 'joe@bitworking.org'
+__license__ = 'MIT License'
+__credits__ = ''
+
+
+def parse_mime_type(mime_type):
+    """Parses a mime-type into its component parts.
+
+    Carves up a mime-type and returns a tuple of the (type, subtype, params)
+    where 'params' is a dictionary of all the parameters for the media range.
+    For example, the media range 'application/xhtml;q=0.5' would get parsed
+    into:
+
+       ('application', 'xhtml', {'q', '0.5'})
+       """
+    parts = mime_type.split(';')
+    params = dict([tuple([s.strip() for s in param.split('=', 1)])\
+            for param in parts[1:]
+                  ])
+    full_type = parts[0].strip()
+    # Java URLConnection class sends an Accept header that includes a
+    # single '*'. Turn it into a legal wildcard.
+    if full_type == '*':
+        full_type = '*/*'
+    (type, subtype) = full_type.split('/')
+
+    return (type.strip(), subtype.strip(), params)
+
+
+def parse_media_range(range):
+    """Parse a media-range into its component parts.
+
+    Carves up a media range and returns a tuple of the (type, subtype,
+    params) where 'params' is a dictionary of all the parameters for the media
+    range.  For example, the media range 'application/*;q=0.5' would get parsed
+    into:
+
+       ('application', '*', {'q', '0.5'})
+
+    In addition this function also guarantees that there is a value for 'q'
+    in the params dictionary, filling it in with a proper default if
+    necessary.
+    """
+    (type, subtype, params) = parse_mime_type(range)
+    if not params.has_key('q') or not params['q'] or \
+            not float(params['q']) or float(params['q']) > 1\
+            or float(params['q']) < 0:
+        params['q'] = '1'
+
+    return (type, subtype, params)
+
+
+def fitness_and_quality_parsed(mime_type, parsed_ranges):
+    """Find the best match for a mime-type amongst parsed media-ranges.
+
+    Find the best match for a given mime-type against a list of media_ranges
+    that have already been parsed by parse_media_range(). Returns a tuple of
+    the fitness value and the value of the 'q' quality parameter of the best
+    match, or (-1, 0) if no match was found. Just as for quality_parsed(),
+    'parsed_ranges' must be a list of parsed media ranges.
+    """
+    best_fitness = -1
+    best_fit_q = 0
+    (target_type, target_subtype, target_params) =\
+            parse_media_range(mime_type)
+    for (type, subtype, params) in parsed_ranges:
+        type_match = (type == target_type or\
+                      type == '*' or\
+                      target_type == '*')
+        subtype_match = (subtype == target_subtype or\
+                         subtype == '*' or\
+                         target_subtype == '*')
+        if type_match and subtype_match:
+            param_matches = reduce(lambda x, y: x + y, [1 for (key, value) in \
+                    target_params.iteritems() if key != 'q' and \
+                    params.has_key(key) and value == params[key]], 0)
+            fitness = (type == target_type) and 100 or 0
+            fitness += (subtype == target_subtype) and 10 or 0
+            fitness += param_matches
+            if fitness > best_fitness:
+                best_fitness = fitness
+                best_fit_q = params['q']
+
+    return best_fitness, float(best_fit_q)
+
+
+def quality_parsed(mime_type, parsed_ranges):
+    """Find the best match for a mime-type amongst parsed media-ranges.
+
+    Find the best match for a given mime-type against a list of media_ranges
+    that have already been parsed by parse_media_range(). Returns the 'q'
+    quality parameter of the best match, 0 if no match was found. This function
+    bahaves the same as quality() except that 'parsed_ranges' must be a list of
+    parsed media ranges.
+    """
+
+    return fitness_and_quality_parsed(mime_type, parsed_ranges)[1]
+
+
+def quality(mime_type, ranges):
+    """Return the quality ('q') of a mime-type against a list of media-ranges.
+
+    Returns the quality 'q' of a mime-type when compared against the
+    media-ranges in ranges. For example:
+
+    >>> quality('text/html','text/*;q=0.3, text/html;q=0.7,
+                  text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5')
+    0.7
+
+    """
+    parsed_ranges = [parse_media_range(r) for r in ranges.split(',')]
+
+    return quality_parsed(mime_type, parsed_ranges)
+
+
+def best_match(supported, header):
+    """Return mime-type with the highest quality ('q') from list of candidates.
+
+    Takes a list of supported mime-types and finds the best match for all the
+    media-ranges listed in header. The value of header must be a string that
+    conforms to the format of the HTTP Accept: header. The value of 'supported'
+    is a list of mime-types. The list of supported mime-types should be sorted
+    in order of increasing desirability, in case of a situation where there is
+    a tie.
+
+    >>> best_match(['application/xbel+xml', 'text/xml'],
+                   'text/*;q=0.5,*/*; q=0.1')
+    'text/xml'
+    """
+    split_header = _filter_blank(header.split(','))
+    parsed_header = [parse_media_range(r) for r in split_header]
+    weighted_matches = []
+    pos = 0
+    for mime_type in supported:
+        weighted_matches.append((fitness_and_quality_parsed(mime_type,
+                                 parsed_header), pos, mime_type))
+        pos += 1
+    weighted_matches.sort()
+
+    return weighted_matches[-1][0][1] and weighted_matches[-1][2] or ''
+
+
+def _filter_blank(i):
+    for s in i:
+        if s.strip():
+            yield s

tests/data/zoo.json

@@ -258,6 +258,22 @@
      },
      "response": {
       "$ref": "Animal"
+     },
+     "mediaUpload": {
+      "accept": [
+       "image/png"
+      ],
+      "maxSize": "1KB",
+      "protocols": {
+       "simple": {
+        "multipart": true,
+        "path": "upload/activities/{userId}/@self"
+       },
+       "resumable": {
+        "multipart": true,
+        "path": "upload/activities/{userId}/@self"
+       }
+      }
      }
     },
     "list": {

tests/test_discovery.py

@@ -38,6 +38,8 @@ from apiclient.http import tunnel_patch
 from apiclient.http import HttpMockSequence
 from apiclient.errors import HttpError
 from apiclient.errors import InvalidJsonError
+from apiclient.errors import MediaUploadSizeError
+from apiclient.errors import UnacceptableMimeTypeError
 
 
 DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
@@ -195,6 +197,66 @@ class Discovery(unittest.TestCase):
     q = parse_qs(parsed[4])
     self.assertEqual(q['q'], ['foo'])
 
+  def test_simple_media_uploads(self):
+    self.http = HttpMock(datafile('zoo.json'), {'status': '200'})
+    zoo = build('zoo', 'v1', self.http)
+    doc = getattr(zoo.animals().insert, '__doc__')
+    self.assertTrue('media_body' in doc)
+
+  def test_simple_media_raise_correct_exceptions(self):
+    self.http = HttpMock(datafile('zoo.json'), {'status': '200'})
+    zoo = build('zoo', 'v1', self.http)
+
+    try:
+      zoo.animals().insert(media_body=datafile('smiley.png'))
+      self.fail("should throw exception if media is too large.")
+    except MediaUploadSizeError:
+      pass
+
+    try:
+      zoo.animals().insert(media_body=datafile('small.jpg'))
+      self.fail("should throw exception if mimetype is unacceptable.")
+    except UnacceptableMimeTypeError:
+      pass
+
+  def test_simple_media_good_upload(self):
+    self.http = HttpMock(datafile('zoo.json'), {'status': '200'})
+    zoo = build('zoo', 'v1', self.http)
+
+    request = zoo.animals().insert(media_body=datafile('small.png'))
+    self.assertEquals('image/png', request.headers['content-type'])
+    self.assertEquals('PNG', request.body[1:4])
+
+  def test_multipart_media_raise_correct_exceptions(self):
+    self.http = HttpMock(datafile('zoo.json'), {'status': '200'})
+    zoo = build('zoo', 'v1', self.http)
+
+    try:
+      zoo.animals().insert(media_body=datafile('smiley.png'), body={})
+      self.fail("should throw exception if media is too large.")
+    except MediaUploadSizeError:
+      pass
+
+    try:
+      zoo.animals().insert(media_body=datafile('small.jpg'), body={})
+      self.fail("should throw exception if mimetype is unacceptable.")
+    except UnacceptableMimeTypeError:
+      pass
+
+  def test_multipart_media_good_upload(self):
+    self.http = HttpMock(datafile('zoo.json'), {'status': '200'})
+    zoo = build('zoo', 'v1', self.http)
+
+    request = zoo.animals().insert(media_body=datafile('small.png'), body={})
+    self.assertTrue(request.headers['content-type'].startswith('multipart/related'))
+    self.assertEquals('--==', request.body[0:4])
+
+  def test_media_capable_method_without_media(self):
+    self.http = HttpMock(datafile('zoo.json'), {'status': '200'})
+    zoo = build('zoo', 'v1', self.http)
+
+    request = zoo.animals().insert(body={})
+    self.assertTrue(request.headers['content-type'], 'application/json')
 
 class Next(unittest.TestCase):
   def test_next_for_people_liked(self):