# Copyright (C) 2010 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Client for discovery based APIs A client library for Google's discovery based APIs. """ __author__ = 'jcgregorio@google.com (Joe Gregorio)' __all__ = [ 'build', 'build_from_document' 'fix_method_name', 'key2param' ] import copy import httplib2 import logging import os import random import re import uritemplate import urllib import urlparse import mimeparse import mimetypes try: from urlparse import parse_qsl except ImportError: from cgi import parse_qsl from apiclient.errors import HttpError from apiclient.errors import InvalidJsonError from apiclient.errors import MediaUploadSizeError from apiclient.errors import UnacceptableMimeTypeError from apiclient.errors import UnknownApiNameOrVersion from apiclient.errors import UnknownLinkType from apiclient.http import HttpRequest from apiclient.http import MediaFileUpload from apiclient.http import MediaUpload from apiclient.model import JsonModel from apiclient.model import MediaModel from apiclient.model import RawModel from apiclient.schema import Schemas from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart from oauth2client.anyjson import simplejson logger = logging.getLogger(__name__) URITEMPLATE = re.compile('{[^}]*}') VARNAME = re.compile('[a-zA-Z0-9_-]+') DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/' '{api}/{apiVersion}/rest') DEFAULT_METHOD_DOC = 'A description of how to use this function' # Parameters accepted by the stack, but not visible via discovery. STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict'] # Python reserved words. RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or', 'pass', 'print', 'raise', 'return', 'try', 'while', 'body'] def fix_method_name(name): """Fix method names to avoid reserved word conflicts. Args: name: string, method name. Returns: The name with a '_' prefixed if the name is a reserved word. """ if name in RESERVED_WORDS: return name + '_' else: return name def _add_query_parameter(url, name, value): """Adds a query parameter to a url. Replaces the current value if it already exists in the URL. Args: url: string, url to add the query parameter to. name: string, query parameter name. value: string, query parameter value. Returns: Updated query parameter. Does not update the url if value is None. """ if value is None: return url else: parsed = list(urlparse.urlparse(url)) q = dict(parse_qsl(parsed[4])) q[name] = value parsed[4] = urllib.urlencode(q) return urlparse.urlunparse(parsed) def key2param(key): """Converts key names into parameter names. For example, converting "max-results" -> "max_results" Args: key: string, the method key name. Returns: A safe method name based on the key name. """ result = [] key = list(key) if not key[0].isalpha(): result.append('x') for c in key: if c.isalnum(): result.append(c) else: result.append('_') return ''.join(result) def build(serviceName, version, http=None, discoveryServiceUrl=DISCOVERY_URI, developerKey=None, model=None, requestBuilder=HttpRequest): """Construct a Resource for interacting with an API. Construct a Resource object for interacting with an API. The serviceName and version are the names from the Discovery service. Args: serviceName: string, name of the service. version: string, the version of the service. http: httplib2.Http, An instance of httplib2.Http or something that acts like it that HTTP requests will be made through. discoveryServiceUrl: string, a URI Template that points to the location of the discovery service. It should have two parameters {api} and {apiVersion} that when filled in produce an absolute URI to the discovery document for that service. developerKey: string, key obtained from https://code.google.com/apis/console. model: apiclient.Model, converts to and from the wire format. requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP request. Returns: A Resource object with methods for interacting with the service. """ params = { 'api': serviceName, 'apiVersion': version } if http is None: http = httplib2.Http() requested_url = uritemplate.expand(discoveryServiceUrl, params) # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment # variable that contains the network address of the client sending the # request. If it exists then add that to the request for the discovery # document to avoid exceeding the quota on discovery requests. if 'REMOTE_ADDR' in os.environ: requested_url = _add_query_parameter(requested_url, 'userIp', os.environ['REMOTE_ADDR']) logger.info('URL being requested: %s' % requested_url) resp, content = http.request(requested_url) if resp.status == 404: raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName, version)) if resp.status >= 400: raise HttpError(resp, content, requested_url) try: service = simplejson.loads(content) except ValueError, e: logger.error('Failed to parse as JSON: ' + content) raise InvalidJsonError() return build_from_document(content, discoveryServiceUrl, http=http, developerKey=developerKey, model=model, requestBuilder=requestBuilder) def build_from_document( service, base=None, future=None, http=None, developerKey=None, model=None, requestBuilder=HttpRequest): """Create a Resource for interacting with an API. Same as `build()`, but constructs the Resource object from a discovery document that is it given, as opposed to retrieving one over HTTP. Args: service: string, discovery document. base: string, base URI for all HTTP requests, usually the discovery URI. This parameter is no longer used as rootUrl and servicePath are included within the discovery document. (deprecated) future: string, discovery document with future capabilities (deprecated). http: httplib2.Http, An instance of httplib2.Http or something that acts like it that HTTP requests will be made through. developerKey: string, Key for controlling API usage, generated from the API Console. model: Model class instance that serializes and de-serializes requests and responses. requestBuilder: Takes an http request and packages it up to be executed. Returns: A Resource object with methods for interacting with the service. """ # future is no longer used. future = {} service = simplejson.loads(service) base = urlparse.urljoin(service['rootUrl'], service['servicePath']) schema = Schemas(service) if model is None: features = service.get('features', []) model = JsonModel('dataWrapper' in features) resource = _createResource(http, base, model, requestBuilder, developerKey, service, service, schema) return resource def _cast(value, schema_type): """Convert value to a string based on JSON Schema type. See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on JSON Schema. Args: value: any, the value to convert schema_type: string, the type that value should be interpreted as Returns: A string representation of 'value' based on the schema_type. """ if schema_type == 'string': if type(value) == type('') or type(value) == type(u''): return value else: return str(value) elif schema_type == 'integer': return str(int(value)) elif schema_type == 'number': return str(float(value)) elif schema_type == 'boolean': return str(bool(value)).lower() else: if type(value) == type('') or type(value) == type(u''): return value else: return str(value) MULTIPLIERS = { "KB": 2 ** 10, "MB": 2 ** 20, "GB": 2 ** 30, "TB": 2 ** 40, } def _media_size_to_long(maxSize): """Convert a string media size, such as 10GB or 3TB into an integer. Args: maxSize: string, size as a string, such as 2MB or 7GB. Returns: The size as an integer value. """ if len(maxSize) < 2: return 0 units = maxSize[-2:].upper() multiplier = MULTIPLIERS.get(units, 0) if multiplier: return int(maxSize[:-2]) * multiplier else: return int(maxSize) def _createResource(http, baseUrl, model, requestBuilder, developerKey, resourceDesc, rootDesc, schema): """Build a Resource from the API description. Args: http: httplib2.Http, Object to make http requests with. baseUrl: string, base URL for the API. All requests are relative to this URI. model: apiclient.Model, converts to and from the wire format. requestBuilder: class or callable that instantiates an apiclient.HttpRequest object. developerKey: string, key obtained from https://code.google.com/apis/console resourceDesc: object, section of deserialized discovery document that describes a resource. Note that the top level discovery document is considered a resource. rootDesc: object, the entire deserialized discovery document. schema: object, mapping of schema names to schema descriptions. Returns: An instance of Resource with all the methods attached for interacting with that resource. """ class Resource(object): """A class for interacting with a resource.""" def __init__(self): self._http = http self._baseUrl = baseUrl self._model = model self._developerKey = developerKey self._requestBuilder = requestBuilder def createMethod(theclass, methodName, methodDesc, rootDesc): """Creates a method for attaching to a Resource. Args: theclass: type, the class to attach methods to. methodName: string, name of the method to use. methodDesc: object, fragment of deserialized discovery document that describes the method. rootDesc: object, the entire deserialized discovery document. """ methodName = fix_method_name(methodName) pathUrl = methodDesc['path'] httpMethod = methodDesc['httpMethod'] methodId = methodDesc['id'] mediaPathUrl = None accept = [] maxSize = 0 if 'mediaUpload' in methodDesc: mediaUpload = methodDesc['mediaUpload'] mediaPathUrl = (rootDesc['rootUrl'] + 'upload/' + rootDesc['servicePath'] + pathUrl) accept = mediaUpload['accept'] maxSize = _media_size_to_long(mediaUpload.get('maxSize', '')) if 'parameters' not in methodDesc: methodDesc['parameters'] = {} # Add in the parameters common to all methods. for name, desc in rootDesc.get('parameters', {}).iteritems(): methodDesc['parameters'][name] = desc # Add in undocumented query parameters. for name in STACK_QUERY_PARAMETERS: methodDesc['parameters'][name] = { 'type': 'string', 'location': 'query' } if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc: methodDesc['parameters']['body'] = { 'description': 'The request body.', 'type': 'object', 'required': True, } if 'request' in methodDesc: methodDesc['parameters']['body'].update(methodDesc['request']) else: methodDesc['parameters']['body']['type'] = 'object' if 'mediaUpload' in methodDesc: methodDesc['parameters']['media_body'] = { 'description': 'The filename of the media request body, or an instance of a ' 'MediaUpload object.', 'type': 'string', 'required': False, } if 'body' in methodDesc['parameters']: methodDesc['parameters']['body']['required'] = False argmap = {} # Map from method parameter name to query parameter name required_params = [] # Required parameters repeated_params = [] # Repeated parameters pattern_params = {} # Parameters that must match a regex query_params = [] # Parameters that will be used in the query string path_params = {} # Parameters that will be used in the base URL param_type = {} # The type of the parameter enum_params = {} # Allowable enumeration values for each parameter if 'parameters' in methodDesc: for arg, desc in methodDesc['parameters'].iteritems(): param = key2param(arg) argmap[param] = arg if desc.get('pattern', ''): pattern_params[param] = desc['pattern'] if desc.get('enum', ''): enum_params[param] = desc['enum'] if desc.get('required', False): required_params.append(param) if desc.get('repeated', False): repeated_params.append(param) if desc.get('location') == 'query': query_params.append(param) if desc.get('location') == 'path': path_params[param] = param param_type[param] = desc.get('type', 'string') for match in URITEMPLATE.finditer(pathUrl): for namematch in VARNAME.finditer(match.group(0)): name = key2param(namematch.group(0)) path_params[name] = name if name in query_params: query_params.remove(name) def method(self, **kwargs): # Don't bother with doc string, it will be over-written by createMethod. for name in kwargs.iterkeys(): if name not in argmap: raise TypeError('Got an unexpected keyword argument "%s"' % name) # Remove args that have a value of None. keys = kwargs.keys() for name in keys: if kwargs[name] is None: del kwargs[name] for name in required_params: if name not in kwargs: raise TypeError('Missing required parameter "%s"' % name) for name, regex in pattern_params.iteritems(): if name in kwargs: if isinstance(kwargs[name], basestring): pvalues = [kwargs[name]] else: pvalues = kwargs[name] for pvalue in pvalues: if re.match(regex, pvalue) is None: raise TypeError( 'Parameter "%s" value "%s" does not match the pattern "%s"' % (name, pvalue, regex)) for name, enums in enum_params.iteritems(): if name in kwargs: # We need to handle the case of a repeated enum # name differently, since we want to handle both # arg='value' and arg=['value1', 'value2'] if (name in repeated_params and not isinstance(kwargs[name], basestring)): values = kwargs[name] else: values = [kwargs[name]] for value in values: if value not in enums: raise TypeError( 'Parameter "%s" value "%s" is not an allowed value in "%s"' % (name, value, str(enums))) actual_query_params = {} actual_path_params = {} for key, value in kwargs.iteritems(): to_type = param_type.get(key, 'string') # For repeated parameters we cast each member of the list. if key in repeated_params and type(value) == type([]): cast_value = [_cast(x, to_type) for x in value] else: cast_value = _cast(value, to_type) if key in query_params: actual_query_params[argmap[key]] = cast_value if key in path_params: actual_path_params[argmap[key]] = cast_value body_value = kwargs.get('body', None) media_filename = kwargs.get('media_body', None) if self._developerKey: actual_query_params['key'] = self._developerKey model = self._model if methodName.endswith('_media'): model = MediaModel() elif 'response' not in methodDesc: model = RawModel() headers = {} headers, params, query, body = model.request(headers, actual_path_params, actual_query_params, body_value) expanded_url = uritemplate.expand(pathUrl, params) url = urlparse.urljoin(self._baseUrl, expanded_url + query) resumable = None multipart_boundary = '' if media_filename: # Ensure we end up with a valid MediaUpload object. if isinstance(media_filename, basestring): (media_mime_type, encoding) = mimetypes.guess_type(media_filename) if media_mime_type is None: raise UnknownFileType(media_filename) if not mimeparse.best_match([media_mime_type], ','.join(accept)): raise UnacceptableMimeTypeError(media_mime_type) media_upload = MediaFileUpload(media_filename, media_mime_type) elif isinstance(media_filename, MediaUpload): media_upload = media_filename else: raise TypeError('media_filename must be str or MediaUpload.') # Check the maxSize if maxSize > 0 and media_upload.size() > maxSize: raise MediaUploadSizeError("Media larger than: %s" % maxSize) # Use the media path uri for media uploads expanded_url = uritemplate.expand(mediaPathUrl, params) url = urlparse.urljoin(self._baseUrl, expanded_url + query) if media_upload.resumable(): url = _add_query_parameter(url, 'uploadType', 'resumable') if media_upload.resumable(): # This is all we need to do for resumable, if the body exists it gets # sent in the first request, otherwise an empty body is sent. resumable = media_upload else: # A non-resumable upload if body is None: # This is a simple media upload headers['content-type'] = media_upload.mimetype() body = media_upload.getbytes(0, media_upload.size()) url = _add_query_parameter(url, 'uploadType', 'media') else: # This is a multipart/related upload. msgRoot = MIMEMultipart('related') # msgRoot should not write out it's own headers setattr(msgRoot, '_write_headers', lambda self: None) # attach the body as one part msg = MIMENonMultipart(*headers['content-type'].split('/')) msg.set_payload(body) msgRoot.attach(msg) # attach the media as the second part msg = MIMENonMultipart(*media_upload.mimetype().split('/')) msg['Content-Transfer-Encoding'] = 'binary' payload = media_upload.getbytes(0, media_upload.size()) msg.set_payload(payload) msgRoot.attach(msg) body = msgRoot.as_string() multipart_boundary = msgRoot.get_boundary() headers['content-type'] = ('multipart/related; ' 'boundary="%s"') % multipart_boundary url = _add_query_parameter(url, 'uploadType', 'multipart') logger.info('URL being requested: %s' % url) return self._requestBuilder(self._http, model.response, url, method=httpMethod, body=body, headers=headers, methodId=methodId, resumable=resumable) docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n'] if len(argmap) > 0: docs.append('Args:\n') # Skip undocumented params and params common to all methods. skip_parameters = rootDesc.get('parameters', {}).keys() skip_parameters.extend(STACK_QUERY_PARAMETERS) all_args = argmap.keys() args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])] # Move body to the front of the line. if 'body' in all_args: args_ordered.append('body') for name in all_args: if name not in args_ordered: args_ordered.append(name) for arg in args_ordered: if arg in skip_parameters: continue repeated = '' if arg in repeated_params: repeated = ' (repeated)' required = '' if arg in required_params: required = ' (required)' paramdesc = methodDesc['parameters'][argmap[arg]] paramdoc = paramdesc.get('description', 'A parameter') if '$ref' in paramdesc: docs.append( (' %s: object, %s%s%s\n The object takes the' ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated, schema.prettyPrintByName(paramdesc['$ref']))) else: paramtype = paramdesc.get('type', 'string') docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required, repeated)) enum = paramdesc.get('enum', []) enumDesc = paramdesc.get('enumDescriptions', []) if enum and enumDesc: docs.append(' Allowed values\n') for (name, desc) in zip(enum, enumDesc): docs.append(' %s - %s\n' % (name, desc)) if 'response' in methodDesc: if methodName.endswith('_media'): docs.append('\nReturns:\n The media object as a string.\n\n ') else: docs.append('\nReturns:\n An object of the form:\n\n ') docs.append(schema.prettyPrintSchema(methodDesc['response'])) setattr(method, '__doc__', ''.join(docs)) setattr(theclass, methodName, method) def createNextMethod(theclass, methodName, methodDesc, rootDesc): """Creates any _next methods for attaching to a Resource. The _next methods allow for easy iteration through list() responses. Args: theclass: type, the class to attach methods to. methodName: string, name of the method to use. methodDesc: object, fragment of deserialized discovery document that describes the method. rootDesc: object, the entire deserialized discovery document. """ methodName = fix_method_name(methodName) methodId = methodDesc['id'] + '.next' def methodNext(self, previous_request, previous_response): """Retrieves the next page of results. Args: previous_request: The request for the previous page. (required) previous_response: The response from the request for the previous page. (required) Returns: A request object that you can call 'execute()' on to request the next page. Returns None if there are no more items in the collection. """ # Retrieve nextPageToken from previous_response # Use as pageToken in previous_request to create new request. if 'nextPageToken' not in previous_response: return None request = copy.copy(previous_request) pageToken = previous_response['nextPageToken'] parsed = list(urlparse.urlparse(request.uri)) q = parse_qsl(parsed[4]) # Find and remove old 'pageToken' value from URI newq = [(key, value) for (key, value) in q if key != 'pageToken'] newq.append(('pageToken', pageToken)) parsed[4] = urllib.urlencode(newq) uri = urlparse.urlunparse(parsed) request.uri = uri logger.info('URL being requested: %s' % uri) return request setattr(theclass, methodName, methodNext) # Add basic methods to Resource if 'methods' in resourceDesc: for methodName, methodDesc in resourceDesc['methods'].iteritems(): createMethod(Resource, methodName, methodDesc, rootDesc) # Add in _media methods. The functionality of the attached method will # change when it sees that the method name ends in _media. if methodDesc.get('supportsMediaDownload', False): createMethod(Resource, methodName + '_media', methodDesc, rootDesc) # Add in nested resources if 'resources' in resourceDesc: def createResourceMethod(theclass, methodName, methodDesc, rootDesc): """Create a method on the Resource to access a nested Resource. Args: theclass: type, the class to attach methods to. methodName: string, name of the method to use. methodDesc: object, fragment of deserialized discovery document that describes the method. rootDesc: object, the entire deserialized discovery document. """ methodName = fix_method_name(methodName) def methodResource(self): return _createResource(self._http, self._baseUrl, self._model, self._requestBuilder, self._developerKey, methodDesc, rootDesc, schema) setattr(methodResource, '__doc__', 'A collection resource.') setattr(methodResource, '__is_resource__', True) setattr(theclass, methodName, methodResource) for methodName, methodDesc in resourceDesc['resources'].iteritems(): createResourceMethod(Resource, methodName, methodDesc, rootDesc) # Add _next() methods # Look for response bodies in schema that contain nextPageToken, and methods # that take a pageToken parameter. if 'methods' in resourceDesc: for methodName, methodDesc in resourceDesc['methods'].iteritems(): if 'response' in methodDesc: responseSchema = methodDesc['response'] if '$ref' in responseSchema: responseSchema = schema.get(responseSchema['$ref']) hasNextPageToken = 'nextPageToken' in responseSchema.get('properties', {}) hasPageToken = 'pageToken' in methodDesc.get('parameters', {}) if hasNextPageToken and hasPageToken: createNextMethod(Resource, methodName + '_next', resourceDesc['methods'][methodName], methodName) return Resource()