diff --git a/apiclient/contrib/__init__.py b/apiclient/contrib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apiclient/ext/__init__.py b/apiclient/ext/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apiclient/ext/appengine.py b/apiclient/ext/appengine.py new file mode 100644 index 0000000..a780d0e --- /dev/null +++ b/apiclient/ext/appengine.py @@ -0,0 +1,90 @@ +# Copyright (C) 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for Google App Engine + +Utilities for making it easier to use the +Google API Client for Python on Google App Engine. +""" + +__author__ = 'jcgregorio@google.com (Joe Gregorio)' + +import pickle + +from google.appengine.ext import db +from apiclient.oauth import OAuthCredentials +from apiclient.oauth import FlowThreeLegged + + +class FlowThreeLeggedProperty(db.Property): + """Utility property that allows easy + storage and retreival of an + apiclient.oauth.FlowThreeLegged""" + + # Tell what the user type is. + data_type = FlowThreeLegged + + # For writing to datastore. + def get_value_for_datastore(self, model_instance): + flow = super(FlowThreeLeggedProperty, + self).get_value_for_datastore(model_instance) + return db.Blob(pickle.dumps(flow)) + + # For reading from datastore. + def make_value_from_datastore(self, value): + if value is None: + return None + return pickle.loads(value) + + def validate(self, value): + if value is not None and not isinstance(value, FlowThreeLegged): + raise BadValueError('Property %s must be convertible ' + 'to a FlowThreeLegged instance (%s)' % + (self.name, value)) + return super(FlowThreeLeggedProperty, self).validate(value) + + def empty(self, value): + return not value + + +class OAuthCredentialsProperty(db.Property): + """Utility property that allows easy + storage and retrieval of + apiclient.oath.OAuthCredentials + """ + + # Tell what the user type is. + data_type = OAuthCredentials + + # For writing to datastore. + def get_value_for_datastore(self, model_instance): + cred = super(OAuthCredentialsProperty, + self).get_value_for_datastore(model_instance) + return db.Blob(pickle.dumps(cred)) + + # For reading from datastore. + def make_value_from_datastore(self, value): + if value is None: + return None + return pickle.loads(value) + + def validate(self, value): + if value is not None and not isinstance(value, OAuthCredentials): + raise BadValueError('Property %s must be convertible ' + 'to an OAuthCredentials instance (%s)' % + (self.name, value)) + return super(OAuthCredentialsProperty, self).validate(value) + + def empty(self, value): + return not value diff --git a/apiclient/oauth.py b/apiclient/oauth.py index c1908d1..63f896b 100644 --- a/apiclient/oauth.py +++ b/apiclient/oauth.py @@ -10,30 +10,25 @@ Utilities for making it easier to work with OAuth. __author__ = 'jcgregorio@google.com (Joe Gregorio)' import copy -import urllib +import httplib2 import oauth2 as oauth +import urllib +import logging try: from urlparse import parse_qs, parse_qsl except ImportError: from cgi import parse_qs, parse_qsl + + class MissingParameter(Exception): pass -def abstract(): + +def _abstract(): raise NotImplementedError("You need to override this function") -class TokenStore(object): - def get(user, service): - """Returns an oauth.Token based on the (user, service) returning - None if there is no Token for that (user, service). - """ - abstract() - - def set(user, service, token): - abstract() - buzz_discovery = { 'required': ['domain', 'scope'], 'request': { @@ -50,7 +45,19 @@ buzz_discovery = { }, } + def _oauth_uri(name, discovery, params): + """Look up the OAuth UR from the discovery + document and add query parameters based on + params. + + name - The name of the OAuth URI to lookup, one + of 'request', 'access', or 'authorize'. + discovery - Portion of discovery document the describes + the OAuth endpoints. + params - Dictionary that is used to form the query parameters + for the specified URI. + """ if name not in ['request', 'access', 'authorize']: raise KeyError(name) keys = [] @@ -62,9 +69,100 @@ def _oauth_uri(name, discovery, params): query[key] = params[key] return discovery[name]['url'] + '?' + urllib.urlencode(query) -class Flow3LO(object): + +class Credentials(object): + """Base class for all Credentials objects. + + Subclasses must define an authorize() method + that applies the credentials to an HTTP transport. + """ + + def authorize(self, http): + """Take an httplib2.Http instance (or equivalent) and + authorizes it for the set of credentials, usually by + replacing http.request() with a method that adds in + the appropriate headers and then delegates to the original + Http.request() method. + """ + _abstract() + + +class OAuthCredentials(Credentials): + """Credentials object for OAuth 1.0a + """ + + def __init__(self, consumer, token, user_agent): + """ + consumer - An instance of oauth.Consumer. + token - An instance of oauth.Token constructed with + the access token and secret. + user_agent - The HTTP User-Agent to provide for this application. + """ + self.consumer = consumer + self.token = token + self.user_agent = user_agent + + def authorize(self, http): + """ + Args: + http - An instance of httplib2.Http + or something that acts like it. + + Returns: + A modified instance of http that was passed in. + + Example: + + h = httplib2.Http() + h = credentials.authorize(h) + + You can't create a new OAuth + subclass of httplib2.Authenication because + it never gets passed the absolute URI, which is + needed for signing. So instead we have to overload + 'request' with a closure that adds in the + Authorization header and then calls the original version + of 'request()'. + """ + request_orig = http.request + signer = oauth.SignatureMethod_HMAC_SHA1() + + # The closure that will replace 'httplib2.Http.request'. + def new_request(uri, method="GET", body=None, headers=None, + redirections=httplib2.DEFAULT_MAX_REDIRECTS, + connection_type=None): + """Modify the request headers to add the appropriate + Authorization header.""" + req = oauth.Request.from_consumer_and_token( + self.consumer, self.token, http_method=method, http_url=uri) + req.sign_request(signer, self.consumer, self.token) + if headers == None: + headers = {} + headers.update(req.to_header()) + if 'user-agent' not in headers: + headers['user-agent'] = self.user_agent + return request_orig(uri, method, body, headers, + redirections, connection_type) + + http.request = new_request + return http + + +class FlowThreeLegged(object): + """Does the Three Legged Dance for OAuth 1.0a. + """ + def __init__(self, discovery, consumer_key, consumer_secret, user_agent, **kwargs): + """ + discovery - Section of the API discovery document that describes + the OAuth endpoints. + consumer_key - OAuth consumer key + consumer_secret - OAuth consumer secret + user_agent - The HTTP User-Agent that identifies the application. + **kwargs - The keyword arguments are all optional and required + parameters for the OAuth calls. + """ self.discovery = discovery self.consumer_key = consumer_key self.consumer_secret = consumer_secret @@ -75,14 +173,17 @@ class Flow3LO(object): if key not in self.params: raise MissingParameter('Required parameter %s not supplied' % key) - def step1(self, oauth_callback='oob'): + def step1_get_authorize_url(self, oauth_callback='oob'): """Returns a URI to redirect to the provider. - If oauth_callback is 'oob' then the next call - should be to step2_pin, otherwise oauth_callback - is a URI and the next call should be to - step2_callback() with the query parameters - received at that callback. + oauth_callback - Either the string 'oob' for a non-web-based application, + or a URI that handles the callback from the authorization + server. + + If oauth_callback is 'oob' then pass in the + generated verification code to step2_exchange, + otherwise pass in the query parameters received + at the callback uri to step2_exchange. """ consumer = oauth.Consumer(self.consumer_key, self.consumer_secret) client = oauth.Client(consumer) @@ -93,10 +194,11 @@ class Flow3LO(object): } body = urllib.urlencode({'oauth_callback': oauth_callback}) uri = _oauth_uri('request', self.discovery, self.params) + resp, content = client.request(uri, 'POST', headers=headers, body=body) if resp['status'] != '200': - print content + logging.error('Failed to retrieve temporary authorization: %s' % content) raise Exception('Invalid response %s.' % resp['status']) self.request_token = dict(parse_qsl(content)) @@ -104,15 +206,24 @@ class Flow3LO(object): auth_params = copy.copy(self.params) auth_params['oauth_token'] = self.request_token['oauth_token'] - uri = _oauth_uri('authorize', self.discovery, auth_params) - return uri + return _oauth_uri('authorize', self.discovery, auth_params) - def step2_pin(self, pin): - """Returns an oauth_token and oauth_token_secret in a dictionary""" + def step2_exchange(self, verifier): + """Exhanges an authorized request token + for OAuthCredentials. - token = oauth.Token(self.request_token['oauth_token'], + verifier - either the verifier token, or a dictionary + of the query parameters to the callback, which contains + the oauth_verifier. + """ + + if not (isinstance(verifier, str) or isinstance(verifier, unicode)): + verifier = verifier['oauth_verifier'] + + token = oauth.Token( + self.request_token['oauth_token'], self.request_token['oauth_token_secret']) - token.set_verifier(pin) + token.set_verifier(verifier) consumer = oauth.Consumer(self.consumer_key, self.consumer_secret) client = oauth.Client(consumer, token) @@ -123,8 +234,13 @@ class Flow3LO(object): uri = _oauth_uri('access', self.discovery, self.params) resp, content = client.request(uri, 'POST', headers=headers) - return dict(parse_qsl(content)) + if resp['status'] != '200': + logging.error('Failed to retrieve access token: %s' % content) + raise Exception('Invalid response %s.' % resp['status']) - def step2_callback(self, query_params): - """Returns an access token via oauth.Token""" - pass + oauth_params = dict(parse_qsl(content)) + token = oauth.Token( + oauth_params['oauth_token'], + oauth_params['oauth_token_secret']) + + return OAuthCredentials(consumer, token, self.user_agent) diff --git a/httplib2/__init__.py b/httplib2/__init__.py new file mode 100644 index 0000000..61e9caa --- /dev/null +++ b/httplib2/__init__.py @@ -0,0 +1,1208 @@ +from __future__ import generators +""" +httplib2 + +A caching http interface that supports ETags and gzip +to conserve bandwidth. + +Requires Python 2.3 or later + +Changelog: +2007-08-18, Rick: Modified so it's able to use a socks proxy if needed. + +""" + +__author__ = "Joe Gregorio (joe@bitworking.org)" +__copyright__ = "Copyright 2006, Joe Gregorio" +__contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", + "James Antill", + "Xavier Verges Farrero", + "Jonathan Feinberg", + "Blair Zajac", + "Sam Ruby", + "Louis Nyffenegger"] +__license__ = "MIT" +__version__ = "$Rev$" + +import re +import sys +import email +import email.Utils +import email.Message +import email.FeedParser +import StringIO +import gzip +import zlib +import httplib +import urlparse +import base64 +import os +import copy +import calendar +import time +import random +import errno +# remove depracated warning in python2.6 +try: + from hashlib import sha1 as _sha, md5 as _md5 +except ImportError: + import sha + import md5 + _sha = sha.new + _md5 = md5.new +import hmac +from gettext import gettext as _ +import socket + +try: + from httplib2 import socks +except ImportError: + socks = None + +# Build the appropriate socket wrapper for ssl +try: + import ssl # python 2.6 + _ssl_wrap_socket = ssl.wrap_socket +except ImportError: + def _ssl_wrap_socket(sock, key_file, cert_file): + ssl_sock = socket.ssl(sock, key_file, cert_file) + return httplib.FakeSocket(sock, ssl_sock) + + +if sys.version_info >= (2,3): + from iri2uri import iri2uri +else: + def iri2uri(uri): + return uri + +def has_timeout(timeout): # python 2.6 + if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'): + return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) + return (timeout is not None) + +__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', + 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', + 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', + 'debuglevel'] + + +# The httplib debug level, set to a non-zero value to get debug output +debuglevel = 0 + + +# Python 2.3 support +if sys.version_info < (2,4): + def sorted(seq): + seq.sort() + return seq + +# Python 2.3 support +def HTTPResponse__getheaders(self): + """Return list of (header, value) tuples.""" + if self.msg is None: + raise httplib.ResponseNotReady() + return self.msg.items() + +if not hasattr(httplib.HTTPResponse, 'getheaders'): + httplib.HTTPResponse.getheaders = HTTPResponse__getheaders + +# All exceptions raised here derive from HttpLib2Error +class HttpLib2Error(Exception): pass + +# Some exceptions can be caught and optionally +# be turned back into responses. +class HttpLib2ErrorWithResponse(HttpLib2Error): + def __init__(self, desc, response, content): + self.response = response + self.content = content + HttpLib2Error.__init__(self, desc) + +class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass +class RedirectLimit(HttpLib2ErrorWithResponse): pass +class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass +class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass +class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass + +class RelativeURIError(HttpLib2Error): pass +class ServerNotFoundError(HttpLib2Error): pass + +# Open Items: +# ----------- +# Proxy support + +# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?) + +# Pluggable cache storage (supports storing the cache in +# flat files by default. We need a plug-in architecture +# that can support Berkeley DB and Squid) + +# == Known Issues == +# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator. +# Does not handle Cache-Control: max-stale +# Does not use Age: headers when calculating cache freshness. + + +# The number of redirections to follow before giving up. +# Note that only GET redirects are automatically followed. +# Will also honor 301 requests by saving that info and never +# requesting that URI again. +DEFAULT_MAX_REDIRECTS = 5 + +# Which headers are hop-by-hop headers by default +HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] + +def _get_end2end_headers(response): + hopbyhop = list(HOP_BY_HOP) + hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')]) + return [header for header in response.keys() if header not in hopbyhop] + +URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") + +def parse_uri(uri): + """Parses a URI using the regex given in Appendix B of RFC 3986. + + (scheme, authority, path, query, fragment) = parse_uri(uri) + """ + groups = URI.match(uri).groups() + return (groups[1], groups[3], groups[4], groups[6], groups[8]) + +def urlnorm(uri): + (scheme, authority, path, query, fragment) = parse_uri(uri) + if not scheme or not authority: + raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) + authority = authority.lower() + scheme = scheme.lower() + if not path: + path = "/" + # Could do syntax based normalization of the URI before + # computing the digest. See Section 6.2.2 of Std 66. + request_uri = query and "?".join([path, query]) or path + scheme = scheme.lower() + defrag_uri = scheme + "://" + authority + request_uri + return scheme, authority, request_uri, defrag_uri + + +# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/) +re_url_scheme = re.compile(r'^\w+://') +re_slash = re.compile(r'[?/:|]+') + +def safename(filename): + """Return a filename suitable for the cache. + + Strips dangerous and common characters to create a filename we + can use to store the cache in. + """ + + try: + if re_url_scheme.match(filename): + if isinstance(filename,str): + filename = filename.decode('utf-8') + filename = filename.encode('idna') + else: + filename = filename.encode('idna') + except UnicodeError: + pass + if isinstance(filename,unicode): + filename=filename.encode('utf-8') + filemd5 = _md5(filename).hexdigest() + filename = re_url_scheme.sub("", filename) + filename = re_slash.sub(",", filename) + + # limit length of filename + if len(filename)>200: + filename=filename[:200] + return ",".join((filename, filemd5)) + +NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') +def _normalize_headers(headers): + return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) + +def _parse_cache_control(headers): + retval = {} + if headers.has_key('cache-control'): + parts = headers['cache-control'].split(',') + parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] + parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] + retval = dict(parts_with_args + parts_wo_args) + return retval + +# Whether to use a strict mode to parse WWW-Authenticate headers +# Might lead to bad results in case of ill-formed header value, +# so disabled by default, falling back to relaxed parsing. +# Set to true to turn on, usefull for testing servers. +USE_WWW_AUTH_STRICT_PARSING = 0 + +# In regex below: +# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP +# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space +# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: +# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"? +WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") +WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(? current_age: + retval = "FRESH" + return retval + +def _decompressContent(response, new_content): + content = new_content + try: + encoding = response.get('content-encoding', None) + if encoding in ['gzip', 'deflate']: + if encoding == 'gzip': + content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read() + if encoding == 'deflate': + content = zlib.decompress(content) + response['content-length'] = str(len(content)) + # Record the historical presence of the encoding in a way the won't interfere. + response['-content-encoding'] = response['content-encoding'] + del response['content-encoding'] + except IOError: + content = "" + raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content) + return content + +def _updateCache(request_headers, response_headers, content, cache, cachekey): + if cachekey: + cc = _parse_cache_control(request_headers) + cc_response = _parse_cache_control(response_headers) + if cc.has_key('no-store') or cc_response.has_key('no-store'): + cache.delete(cachekey) + else: + info = email.Message.Message() + for key, value in response_headers.iteritems(): + if key not in ['status','content-encoding','transfer-encoding']: + info[key] = value + + # Add annotations to the cache to indicate what headers + # are variant for this request. + vary = response_headers.get('vary', None) + if vary: + vary_headers = vary.lower().replace(' ', '').split(',') + for header in vary_headers: + key = '-varied-%s' % header + try: + info[key] = request_headers[header] + except KeyError: + pass + + status = response_headers.status + if status == 304: + status = 200 + + status_header = 'status: %d\r\n' % response_headers.status + + header_str = info.as_string() + + header_str = re.sub("\r(?!\n)|(? 0: + service = "cl" + # No point in guessing Base or Spreadsheet + #elif request_uri.find("spreadsheets") > 0: + # service = "wise" + + auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) + resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) + lines = content.split('\n') + d = dict([tuple(line.split("=", 1)) for line in lines if line]) + if resp.status == 403: + self.Auth = "" + else: + self.Auth = d['Auth'] + + def request(self, method, request_uri, headers, content): + """Modify the request headers to add the appropriate + Authorization header.""" + headers['authorization'] = 'GoogleLogin Auth=' + self.Auth + + +AUTH_SCHEME_CLASSES = { + "basic": BasicAuthentication, + "wsse": WsseAuthentication, + "digest": DigestAuthentication, + "hmacdigest": HmacDigestAuthentication, + "googlelogin": GoogleLoginAuthentication +} + +AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] + +class FileCache(object): + """Uses a local directory as a store for cached files. + Not really safe to use if multiple threads or processes are going to + be running on the same cache. + """ + def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior + self.cache = cache + self.safe = safe + if not os.path.exists(cache): + os.makedirs(self.cache) + + def get(self, key): + retval = None + cacheFullPath = os.path.join(self.cache, self.safe(key)) + try: + f = file(cacheFullPath, "rb") + retval = f.read() + f.close() + except IOError: + pass + return retval + + def set(self, key, value): + cacheFullPath = os.path.join(self.cache, self.safe(key)) + f = file(cacheFullPath, "wb") + f.write(value) + f.close() + + def delete(self, key): + cacheFullPath = os.path.join(self.cache, self.safe(key)) + if os.path.exists(cacheFullPath): + os.remove(cacheFullPath) + +class Credentials(object): + def __init__(self): + self.credentials = [] + + def add(self, name, password, domain=""): + self.credentials.append((domain.lower(), name, password)) + + def clear(self): + self.credentials = [] + + def iter(self, domain): + for (cdomain, name, password) in self.credentials: + if cdomain == "" or domain == cdomain: + yield (name, password) + +class KeyCerts(Credentials): + """Identical to Credentials except that + name/password are mapped to key/cert.""" + pass + + +class ProxyInfo(object): + """Collect information required to use a proxy.""" + def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns=None, proxy_user=None, proxy_pass=None): + """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX + constants. For example: + +p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', proxy_port=8000) + """ + self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass = proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass + + def astuple(self): + return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, + self.proxy_user, self.proxy_pass) + + def isgood(self): + return (self.proxy_host != None) and (self.proxy_port != None) + + +class HTTPConnectionWithTimeout(httplib.HTTPConnection): + """HTTPConnection subclass that supports timeouts""" + + def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): + httplib.HTTPConnection.__init__(self, host, port, strict) + self.timeout = timeout + self.proxy_info = proxy_info + + def connect(self): + """Connect to the host and port specified in __init__.""" + # Mostly verbatim from httplib.py. + msg = "getaddrinfo returns an empty list" + for res in socket.getaddrinfo(self.host, self.port, 0, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + if self.proxy_info and self.proxy_info.isgood(): + self.sock = socks.socksocket(af, socktype, proto) + self.sock.setproxy(*self.proxy_info.astuple()) + else: + self.sock = socket.socket(af, socktype, proto) + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # Different from httplib: support timeouts. + if has_timeout(self.timeout): + self.sock.settimeout(self.timeout) + # End of difference from httplib. + if self.debuglevel > 0: + print "connect: (%s, %s)" % (self.host, self.port) + + self.sock.connect(sa) + except socket.error, msg: + if self.debuglevel > 0: + print 'connect fail:', (self.host, self.port) + if self.sock: + self.sock.close() + self.sock = None + continue + break + if not self.sock: + raise socket.error, msg + +class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): + "This class allows communication via SSL." + + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None): + httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, + cert_file=cert_file, strict=strict) + self.timeout = timeout + self.proxy_info = proxy_info + + def connect(self): + "Connect to a host on a given (SSL) port." + + if self.proxy_info and self.proxy_info.isgood(): + sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) + sock.setproxy(*self.proxy_info.astuple()) + else: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if has_timeout(self.timeout): + sock.settimeout(self.timeout) + sock.connect((self.host, self.port)) + self.sock =_ssl_wrap_socket(sock, self.key_file, self.cert_file) + + + +class Http(object): + """An HTTP client that handles: +- all methods +- caching +- ETags +- compression, +- HTTPS +- Basic +- Digest +- WSSE + +and more. + """ + def __init__(self, cache=None, timeout=None, proxy_info=None): + """The value of proxy_info is a ProxyInfo instance. + +If 'cache' is a string then it is used as a directory name +for a disk cache. Otherwise it must be an object that supports +the same interface as FileCache.""" + self.proxy_info = proxy_info + # Map domain name to an httplib connection + self.connections = {} + # The location of the cache, for now a directory + # where cached responses are held. + if cache and isinstance(cache, str): + self.cache = FileCache(cache) + else: + self.cache = cache + + # Name/password + self.credentials = Credentials() + + # Key/cert + self.certificates = KeyCerts() + + # authorization objects + self.authorizations = [] + + # If set to False then no redirects are followed, even safe ones. + self.follow_redirects = True + + # Which HTTP methods do we apply optimistic concurrency to, i.e. + # which methods get an "if-match:" etag header added to them. + self.optimistic_concurrency_methods = ["PUT"] + + # If 'follow_redirects' is True, and this is set to True then + # all redirecs are followed, including unsafe ones. + self.follow_all_redirects = False + + self.ignore_etag = False + + self.force_exception_to_status_code = False + + self.timeout = timeout + + def _auth_from_challenge(self, host, request_uri, headers, response, content): + """A generator that creates Authorization objects + that can be applied to requests. + """ + challenges = _parse_www_authenticate(response, 'www-authenticate') + for cred in self.credentials.iter(host): + for scheme in AUTH_SCHEME_ORDER: + if challenges.has_key(scheme): + yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) + + def add_credentials(self, name, password, domain=""): + """Add a name and password that will be used + any time a request requires authentication.""" + self.credentials.add(name, password, domain) + + def add_certificate(self, key, cert, domain): + """Add a key and cert that will be used + any time a request requires authentication.""" + self.certificates.add(key, cert, domain) + + def clear_credentials(self): + """Remove all the names and passwords + that are used for authentication""" + self.credentials.clear() + self.authorizations = [] + + def _conn_request(self, conn, request_uri, method, body, headers): + for i in range(2): + try: + conn.request(method, request_uri, body, headers) + except socket.gaierror: + conn.close() + raise ServerNotFoundError("Unable to find the server at %s" % conn.host) + except socket.error, e: + if e.errno == errno.ECONNREFUSED: # Connection refused + raise + except httplib.HTTPException: + # Just because the server closed the connection doesn't apparently mean + # that the server didn't send a response. + pass + try: + response = conn.getresponse() + except (socket.error, httplib.HTTPException): + if i == 0: + conn.close() + conn.connect() + continue + else: + raise + else: + content = "" + if method == "HEAD": + response.close() + else: + content = response.read() + response = Response(response) + if method != "HEAD": + content = _decompressContent(response, content) + break + return (response, content) + + + def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): + """Do the actual request using the connection object + and also follow one level of redirects if necessary""" + + auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] + auth = auths and sorted(auths)[0][1] or None + if auth: + auth.request(method, request_uri, headers, body) + + (response, content) = self._conn_request(conn, request_uri, method, body, headers) + + if auth: + if auth.response(response, body): + auth.request(method, request_uri, headers, body) + (response, content) = self._conn_request(conn, request_uri, method, body, headers ) + response._stale_digest = 1 + + if response.status == 401: + for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): + authorization.request(method, request_uri, headers, body) + (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) + if response.status != 401: + self.authorizations.append(authorization) + authorization.response(response, body) + break + + if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): + if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: + # Pick out the location header and basically start from the beginning + # remembering first to strip the ETag header and decrement our 'depth' + if redirections: + if not response.has_key('location') and response.status != 300: + raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content) + # Fix-up relative redirects (which violate an RFC 2616 MUST) + if response.has_key('location'): + location = response['location'] + (scheme, authority, path, query, fragment) = parse_uri(location) + if authority == None: + response['location'] = urlparse.urljoin(absolute_uri, location) + if response.status == 301 and method in ["GET", "HEAD"]: + response['-x-permanent-redirect-url'] = response['location'] + if not response.has_key('content-location'): + response['content-location'] = absolute_uri + _updateCache(headers, response, content, self.cache, cachekey) + if headers.has_key('if-none-match'): + del headers['if-none-match'] + if headers.has_key('if-modified-since'): + del headers['if-modified-since'] + if response.has_key('location'): + location = response['location'] + old_response = copy.deepcopy(response) + if not old_response.has_key('content-location'): + old_response['content-location'] = absolute_uri + redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method + (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) + response.previous = old_response + else: + raise RedirectLimit( _("Redirected more times than rediection_limit allows."), response, content) + elif response.status in [200, 203] and method == "GET": + # Don't cache 206's since we aren't going to handle byte range requests + if not response.has_key('content-location'): + response['content-location'] = absolute_uri + _updateCache(headers, response, content, self.cache, cachekey) + + return (response, content) + + def _normalize_headers(self, headers): + return _normalize_headers(headers) + +# Need to catch and rebrand some exceptions +# Then need to optionally turn all exceptions into status codes +# including all socket.* and httplib.* exceptions. + + + def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): + """ Performs a single HTTP request. +The 'uri' is the URI of the HTTP resource and can begin +with either 'http' or 'https'. The value of 'uri' must be an absolute URI. + +The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. +There is no restriction on the methods allowed. + +The 'body' is the entity body to be sent with the request. It is a string +object. + +Any extra headers that are to be sent with the request should be provided in the +'headers' dictionary. + +The maximum number of redirect to follow before raising an +exception is 'redirections. The default is 5. + +The return value is a tuple of (response, content), the first +being and instance of the 'Response' class, the second being +a string that contains the response entity body. + """ + try: + if headers is None: + headers = {} + else: + headers = self._normalize_headers(headers) + + if not headers.has_key('user-agent'): + headers['user-agent'] = "Python-httplib2/%s" % __version__ + + uri = iri2uri(uri) + + (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) + domain_port = authority.split(":")[0:2] + if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http': + scheme = 'https' + authority = domain_port[0] + + conn_key = scheme+":"+authority + if conn_key in self.connections: + conn = self.connections[conn_key] + else: + if not connection_type: + connection_type = (scheme == 'https') and HTTPSConnectionWithTimeout or HTTPConnectionWithTimeout + certs = list(self.certificates.iter(authority)) + if scheme == 'https' and certs: + conn = self.connections[conn_key] = connection_type(authority, key_file=certs[0][0], + cert_file=certs[0][1], timeout=self.timeout, proxy_info=self.proxy_info) + else: + conn = self.connections[conn_key] = connection_type(authority, timeout=self.timeout, proxy_info=self.proxy_info) + conn.set_debuglevel(debuglevel) + + if method in ["GET", "HEAD"] and 'range' not in headers and 'accept-encoding' not in headers: + headers['accept-encoding'] = 'gzip, deflate' + + info = email.Message.Message() + cached_value = None + if self.cache: + cachekey = defrag_uri + cached_value = self.cache.get(cachekey) + if cached_value: + # info = email.message_from_string(cached_value) + # + # Need to replace the line above with the kludge below + # to fix the non-existent bug not fixed in this + # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html + try: + info, content = cached_value.split('\r\n\r\n', 1) + feedparser = email.FeedParser.FeedParser() + feedparser.feed(info) + info = feedparser.close() + feedparser._parse = None + except IndexError: + self.cache.delete(cachekey) + cachekey = None + cached_value = None + else: + cachekey = None + + if method in self.optimistic_concurrency_methods and self.cache and info.has_key('etag') and not self.ignore_etag and 'if-match' not in headers: + # http://www.w3.org/1999/04/Editing/ + headers['if-match'] = info['etag'] + + if method not in ["GET", "HEAD"] and self.cache and cachekey: + # RFC 2616 Section 13.10 + self.cache.delete(cachekey) + + # Check the vary header in the cache to see if this request + # matches what varies in the cache. + if method in ['GET', 'HEAD'] and 'vary' in info: + vary = info['vary'] + vary_headers = vary.lower().replace(' ', '').split(',') + for header in vary_headers: + key = '-varied-%s' % header + value = info[key] + if headers.get(header, None) != value: + cached_value = None + break + + if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: + if info.has_key('-x-permanent-redirect-url'): + # Should cached permanent redirects be counted in our redirection count? For now, yes. + (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) + response.previous = Response(info) + response.previous.fromcache = True + else: + # Determine our course of action: + # Is the cached entry fresh or stale? + # Has the client requested a non-cached response? + # + # There seems to be three possible answers: + # 1. [FRESH] Return the cache entry w/o doing a GET + # 2. [STALE] Do the GET (but add in cache validators if available) + # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request + entry_disposition = _entry_disposition(info, headers) + + if entry_disposition == "FRESH": + if not cached_value: + info['status'] = '504' + content = "" + response = Response(info) + if cached_value: + response.fromcache = True + return (response, content) + + if entry_disposition == "STALE": + if info.has_key('etag') and not self.ignore_etag and not 'if-none-match' in headers: + headers['if-none-match'] = info['etag'] + if info.has_key('last-modified') and not 'last-modified' in headers: + headers['if-modified-since'] = info['last-modified'] + elif entry_disposition == "TRANSPARENT": + pass + + (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) + + if response.status == 304 and method == "GET": + # Rewrite the cache entry with the new end-to-end headers + # Take all headers that are in response + # and overwrite their values in info. + # unless they are hop-by-hop, or are listed in the connection header. + + for key in _get_end2end_headers(response): + info[key] = response[key] + merged_response = Response(info) + if hasattr(response, "_stale_digest"): + merged_response._stale_digest = response._stale_digest + _updateCache(headers, merged_response, content, self.cache, cachekey) + response = merged_response + response.status = 200 + response.fromcache = True + + elif response.status == 200: + content = new_content + else: + self.cache.delete(cachekey) + content = new_content + else: + cc = _parse_cache_control(headers) + if cc.has_key('only-if-cached'): + info['status'] = '504' + response = Response(info) + content = "" + else: + (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) + except Exception, e: + if self.force_exception_to_status_code: + if isinstance(e, HttpLib2ErrorWithResponse): + response = e.response + content = e.content + response.status = 500 + response.reason = str(e) + elif isinstance(e, socket.timeout): + content = "Request Timeout" + response = Response( { + "content-type": "text/plain", + "status": "408", + "content-length": len(content) + }) + response.reason = "Request Timeout" + else: + content = str(e) + response = Response( { + "content-type": "text/plain", + "status": "400", + "content-length": len(content) + }) + response.reason = "Bad Request" + else: + raise + + + return (response, content) + + + +class Response(dict): + """An object more like email.Message than httplib.HTTPResponse.""" + + """Is this response from our local cache""" + fromcache = False + + """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """ + version = 11 + + "Status code returned by server. " + status = 200 + + """Reason phrase returned by server.""" + reason = "Ok" + + previous = None + + def __init__(self, info): + # info is either an email.Message or + # an httplib.HTTPResponse object. + if isinstance(info, httplib.HTTPResponse): + for key, value in info.getheaders(): + self[key.lower()] = value + self.status = info.status + self['status'] = str(self.status) + self.reason = info.reason + self.version = info.version + elif isinstance(info, email.Message.Message): + for key, value in info.items(): + self[key] = value + self.status = int(self['status']) + else: + for key, value in info.iteritems(): + self[key] = value + self.status = int(self.get('status', self.status)) + + + def __getattr__(self, name): + if name == 'dict': + return self + else: + raise AttributeError, name diff --git a/httplib2/iri2uri.py b/httplib2/iri2uri.py new file mode 100644 index 0000000..70667ed --- /dev/null +++ b/httplib2/iri2uri.py @@ -0,0 +1,110 @@ +""" +iri2uri + +Converts an IRI to a URI. + +""" +__author__ = "Joe Gregorio (joe@bitworking.org)" +__copyright__ = "Copyright 2006, Joe Gregorio" +__contributors__ = [] +__version__ = "1.0.0" +__license__ = "MIT" +__history__ = """ +""" + +import urlparse + + +# Convert an IRI to a URI following the rules in RFC 3987 +# +# The characters we need to enocde and escape are defined in the spec: +# +# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD +# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF +# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD +# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD +# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD +# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD +# / %xD0000-DFFFD / %xE1000-EFFFD + +escape_range = [ + (0xA0, 0xD7FF ), + (0xE000, 0xF8FF ), + (0xF900, 0xFDCF ), + (0xFDF0, 0xFFEF), + (0x10000, 0x1FFFD ), + (0x20000, 0x2FFFD ), + (0x30000, 0x3FFFD), + (0x40000, 0x4FFFD ), + (0x50000, 0x5FFFD ), + (0x60000, 0x6FFFD), + (0x70000, 0x7FFFD ), + (0x80000, 0x8FFFD ), + (0x90000, 0x9FFFD), + (0xA0000, 0xAFFFD ), + (0xB0000, 0xBFFFD ), + (0xC0000, 0xCFFFD), + (0xD0000, 0xDFFFD ), + (0xE1000, 0xEFFFD), + (0xF0000, 0xFFFFD ), + (0x100000, 0x10FFFD) +] + +def encode(c): + retval = c + i = ord(c) + for low, high in escape_range: + if i < low: + break + if i >= low and i <= high: + retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')]) + break + return retval + + +def iri2uri(uri): + """Convert an IRI to a URI. Note that IRIs must be + passed in a unicode strings. That is, do not utf-8 encode + the IRI before passing it into the function.""" + if isinstance(uri ,unicode): + (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) + authority = authority.encode('idna') + # For each character in 'ucschar' or 'iprivate' + # 1. encode as utf-8 + # 2. then %-encode each octet of that utf-8 + uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) + uri = "".join([encode(c) for c in uri]) + return uri + +if __name__ == "__main__": + import unittest + + class Test(unittest.TestCase): + + def test_uris(self): + """Test that URIs are invariant under the transformation.""" + invariant = [ + u"ftp://ftp.is.co.za/rfc/rfc1808.txt", + u"http://www.ietf.org/rfc/rfc2396.txt", + u"ldap://[2001:db8::7]/c=GB?objectClass?one", + u"mailto:John.Doe@example.com", + u"news:comp.infosystems.www.servers.unix", + u"tel:+1-816-555-1212", + u"telnet://192.0.2.16:80/", + u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ] + for uri in invariant: + self.assertEqual(uri, iri2uri(uri)) + + def test_iri(self): + """ Test that the right type of escaping is done for each part of the URI.""" + self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}")) + self.assertEqual("http://bitworking.org/?fred=%E2%98%84", iri2uri(u"http://bitworking.org/?fred=\N{COMET}")) + self.assertEqual("http://bitworking.org/#%E2%98%84", iri2uri(u"http://bitworking.org/#\N{COMET}")) + self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}")) + self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")) + self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"))) + self.assertNotEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode('utf-8'))) + + unittest.main() + + diff --git a/httplib2/socks.py b/httplib2/socks.py new file mode 100644 index 0000000..6f4f020 --- /dev/null +++ b/httplib2/socks.py @@ -0,0 +1,439 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +""" + +Minor modifications made by Christopher Gilbert (http://motomastyle.com/) +for use in PyLoris (http://pyloris.sourceforge.net/) + +Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/) +mainly to merge bug fixes found in Sourceforge + +""" + +import socket +import struct +import sys + +if not hasattr(socket, 'socket'): + raise ImportError("Running on App Engine?") + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 + +_defaultproxy = None + +# Small hack for Python 2.x +if sys.version_info[0] <= 2: + def bytes(obj, enc=None): + return obj + +class ProxyError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class GeneralProxyError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5AuthError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks4Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class HTTPError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + ("request rejected because SOCKS server cannot connect to " + "identd on the client"), + ("request rejected because the client program and identd" + " report different user-ids"), + "unknown error") + + +def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, + username=None, password=None): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype, addr, port, rdns, username, password) + + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=0, _sock=None): + socket.socket.__init__(self, family, type, proto, _sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + + def __decode(self, bytes): + if getattr(bytes, 'decode', False): + try: + bytes = bytes.decode() + except Exception: + pass + return bytes + + def __encode(self, bytes): + if getattr(bytes, 'encode', False): + try: + bytes = bytes.encode() + except Exception: + pass + return bytes + + def __recvall(self, count): + """__recvall(count) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = bytes("") + while len(data) < count: + d = self.recv(count - len(data)) + if not d: + raise GeneralProxyError( + (0, "connection closed unexpectedly")) + data = data + self.__decode(d) + return data + + def sendall(self, bytes): + socket.socket.sendall(self, self.__encode(bytes)) + + def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, + username=None, password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype, addr, port, rdns, username, password) + + def __negotiatesocks5(self, destaddr, destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4] != None) and (self.__proxy[5] != None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall("\x05\x02\x00\x02") + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall("\x05\x01\x00") + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0] != "\x05": + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1] == "\x00": + # No authentication is required + pass + elif chosenauth[1] == "\x02": + # Okay, we need to perform a basic username/password + # authentication. + self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + + chr(len(self.__proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0] != "\x01": + # Bad response + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if authstat[1] != "\x00": + # Authentication failed + self.close() + raise Socks5AuthError((3, _socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == "\xFF": + raise Socks5AuthError((2, _socks5autherrors[2])) + else: + raise GeneralProxyError((1, _generalerrors[1])) + # Now we can request the actual connection + req = "\x05\x01\x00" + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + "\x01" + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3] == True: + # Resolve remotely + ipaddr = None + req = req + "\x03" + chr(len(destaddr)) + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + "\x01" + ipaddr + req = req + self.__decode(struct.pack(">H", destport)) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0] != "\x05": + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + elif resp[1] != "\x00": + # Connection failed + self.close() + if ord(resp[1]) <= 8: + raise Socks5Error((ord(resp[1]), _socks5errors[ord(resp[1])])) + else: + raise Socks5Error((9, _socks5errors[9])) + # Get the bound address/port + elif resp[3] == "\x01": + boundaddr = self.__recvall(4) + elif resp[3] == "\x03": + resp = resp + self.recv(1) + boundaddr = self.__recvall(ord(resp[4])) + else: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + boundport = struct.unpack(">H", bytes(self.__recvall(2), 'utf8'))[0] + self.__proxysockname = boundaddr, boundport + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return socket.socket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self, destaddr, destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3] == True: + ipaddr = "\x00\x00\x00\x01" + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = "\x04\x01" + self.__decode(struct.pack(">H", destport)) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + "\x00" + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv==True: + req = req + destaddr + "\x00" + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0] != "\x00": + # Bad data + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if resp[1] != "\x5A": + # Server returned an error + self.close() + if ord(resp[1]) in (91,92,93): + self.close() + raise Socks4Error((ord(resp[1]), _socks4errors[ord(resp[1])-90])) + else: + raise Socks4Error((94,_socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",bytes(resp[2:4],'utf8'))[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) + else: + self.__proxypeername = (destaddr, destport) + + def __negotiatehttp(self, destaddr, destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if self.__proxy[3] == False: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + self.sendall(("CONNECT %s:%s HTTP/1.1\r\n" + "Host: %s\r\n\r\n") % (addr, destport, destaddr)) + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n") == -1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ", 2) + if statusline[0] not in ("HTTP/1.0", "HTTP/1.1"): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode, statusline[2])) + self.__proxysockname = ("0.0.0.0", 0) + self.__proxypeername = (addr, destport) + + def connect(self, destpair): + """connect(self,despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + # TODO(durin42): seriously? type checking? do we care? + if ((not isinstance(destpair, (list, tuple))) or len(destpair) < 2 + or not isinstance(destpair[0], str) or not isinstance(destpair[1], int)): + raise GeneralProxyError((5, _generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + socket.socket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatesocks5(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + socket.socket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks4(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + socket.socket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatehttp(destpair[0], destpair[1]) + elif self.__proxy[0] == None: + socket.socket.connect(self, (destpair[0], destpair[1])) + else: + raise GeneralProxyError((4, _generalerrors[4])) diff --git a/samples/appengine/apiclient b/samples/appengine/apiclient new file mode 120000 index 0000000..22e4c9c --- /dev/null +++ b/samples/appengine/apiclient @@ -0,0 +1 @@ +../../apiclient/ \ No newline at end of file diff --git a/samples/appengine/app.yaml b/samples/appengine/app.yaml new file mode 100644 index 0000000..66a711a --- /dev/null +++ b/samples/appengine/app.yaml @@ -0,0 +1,8 @@ +application: m-buzz +version: 1 +runtime: python +api_version: 1 + +handlers: +- url: .* + script: main.py diff --git a/samples/appengine/httplib2 b/samples/appengine/httplib2 new file mode 120000 index 0000000..488ae9f --- /dev/null +++ b/samples/appengine/httplib2 @@ -0,0 +1 @@ +../../httplib2/ \ No newline at end of file diff --git a/samples/appengine/index.yaml b/samples/appengine/index.yaml new file mode 100644 index 0000000..a3b9e05 --- /dev/null +++ b/samples/appengine/index.yaml @@ -0,0 +1,11 @@ +indexes: + +# AUTOGENERATED + +# This index.yaml is automatically updated whenever the dev_appserver +# detects that a new type of query is run. If you want to manage the +# index.yaml file manually, remove the above marker line (the line +# saying "# AUTOGENERATED"). If you want to manage some indexes +# manually, move them above the marker line. The index.yaml file is +# automatically uploaded to the admin console when you next deploy +# your application using appcfg.py. diff --git a/samples/appengine/main.py b/samples/appengine/main.py new file mode 100755 index 0000000..6832dda --- /dev/null +++ b/samples/appengine/main.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__author__ = 'jcgregorio@google.com (Joe Gregorio)' + + +import httplib2 +import os + +from apiclient.discovery import build +from apiclient.ext.appengine import FlowThreeLeggedProperty +from apiclient.ext.appengine import OAuthCredentialsProperty +from apiclient.oauth import FlowThreeLegged +from apiclient.oauth import buzz_discovery +from google.appengine.api import users +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template +from google.appengine.ext.webapp import util +from google.appengine.ext.webapp.util import login_required + +STEP2_URI = 'http://m-buzz.appspot.com/auth_return' + + +class Flow(db.Model): + # FlowThreeLegged could also be stored in memcache. + flow = FlowThreeLeggedProperty() + + +class Credentials(db.Model): + credentials = OAuthCredentialsProperty() + + +class MainHandler(webapp.RequestHandler): + + @login_required + def get(self): + user = users.get_current_user() + c = Credentials.get_by_key_name(user.user_id()) + + if c: + http = httplib2.Http() + http = c.credentials.authorize(http) + p = build("buzz", "v1", http=http) + activities = p.activities() + activitylist = activities.list(scope='@self', userId='@me') + path = os.path.join(os.path.dirname(__file__), 'welcome.html') + self.response.out.write(template.render(path, activitylist)) + else: + flow = FlowThreeLegged(buzz_discovery, + consumer_key='anonymous', + consumer_secret='anonymous', + user_agent='google-api-client-python-buzz-webapp/1.0', + domain='anonymous', + scope='https://www.googleapis.com/auth/buzz', + xoauth_displayname='Example Web App') + + authorize_url = flow.step1_get_authorize_url(STEP2_URI) + f = Flow(key_name=user.user_id(), flow=flow) + f.put() + self.redirect(authorize_url) + + +class OAuthHandler(webapp.RequestHandler): + + @login_required + def get(self): + user = users.get_current_user() + f = Flow.get_by_key_name(user.user_id()) + if f: + credentials = f.flow.step2_exchange(self.request.params) + c = Credentials(key_name=user.user_id(), credentials=credentials) + c.put() + f.delete() + self.redirect("/") + else: + pass + + +def main(): + application = webapp.WSGIApplication( + [ + ('/', MainHandler), + ('/auth_return', OAuthHandler) + ], + debug=True) + util.run_wsgi_app(application) + + +if __name__ == '__main__': + main() diff --git a/samples/appengine/oauth2 b/samples/appengine/oauth2 new file mode 120000 index 0000000..af104cc --- /dev/null +++ b/samples/appengine/oauth2 @@ -0,0 +1 @@ +../../oauth2 \ No newline at end of file diff --git a/samples/appengine/simplejson b/samples/appengine/simplejson new file mode 120000 index 0000000..148c7cf --- /dev/null +++ b/samples/appengine/simplejson @@ -0,0 +1 @@ +../../simplejson/ \ No newline at end of file diff --git a/samples/appengine/uritemplate b/samples/appengine/uritemplate new file mode 120000 index 0000000..ce92dcb --- /dev/null +++ b/samples/appengine/uritemplate @@ -0,0 +1 @@ +../../uritemplate/ \ No newline at end of file diff --git a/samples/appengine/welcome.html b/samples/appengine/welcome.html new file mode 100644 index 0000000..421711b --- /dev/null +++ b/samples/appengine/welcome.html @@ -0,0 +1,13 @@ + + + Buzz Stuff + + + + + + diff --git a/samples/cmdline/buzz.py b/samples/cmdline/buzz.py index e6b2ff4..b71f400 100644 --- a/samples/cmdline/buzz.py +++ b/samples/cmdline/buzz.py @@ -3,9 +3,10 @@ # # Copyright 2010 Google Inc. All Rights Reserved. -"""One-line documentation for discovery module. +"""Simple command-line example for Buzz. -A detailed description of discovery. +Command-line application that retrieves the users +latest content and then adds a new entry. """ __author__ = 'jcgregorio@google.com (Joe Gregorio)' @@ -14,72 +15,17 @@ __author__ = 'jcgregorio@google.com (Joe Gregorio)' from apiclient.discovery import build import httplib2 -import oauth2 as oauth -import re -import simplejson - - -def oauth_wrap(consumer, token, http): - """ - Args: - http - An instance of httplib2.Http - or something that acts like it. - - Returns: - A modified instance of http that was passed in. - - Example: - - h = httplib2.Http() - h = oauth_wrap(h) - - Grumble. You can't create a new OAuth - subclass of httplib2.Authenication because - it never gets passed the absolute URI, which is - needed for signing. So instead we have to overload - 'request' with a closure that adds in the - Authorization header and then calls the original version - of 'request()'. - """ - request_orig = http.request - signer = oauth.SignatureMethod_HMAC_SHA1() - - def new_request(uri, method="GET", body=None, headers=None, - redirections=httplib2.DEFAULT_MAX_REDIRECTS, connection_type=None): - """Modify the request headers to add the appropriate - Authorization header.""" - req = oauth.Request.from_consumer_and_token( - consumer, token, http_method=method, http_url=uri) - req.sign_request(signer, consumer, token) - if headers == None: - headers = {} - headers.update(req.to_header()) - headers['user-agent'] = 'jcgregorio-test-client' - return request_orig(uri, method, body, headers, - redirections, connection_type) - - http.request = new_request - return http - - -def get_wrapped_http(): - f = open("oauth_token.dat", "r") - oauth_params = simplejson.loads(f.read()) - - consumer = oauth.Consumer( - oauth_params['consumer_key'], oauth_params['consumer_secret']) - token = oauth.Token( - oauth_params['oauth_token'], oauth_params['oauth_token_secret']) - - # Create a simple monkeypatch for httplib2.Http.request - # just adds in the oauth authorization header and then calls - # the original request(). - http = httplib2.Http() - return oauth_wrap(consumer, token, http) +import pickle def main(): - http = get_wrapped_http() + f = open("oauth_token.dat", "r") + credentials = pickle.loads(f.read()) + f.close() + + http = httplib2.Http() + http = credentials.authorize(http) + p = build("buzz", "v1", http=http) activities = p.activities() activitylist = activities.list(scope='@self', userId='@me') diff --git a/samples/cmdline/three_legged_dance.py b/samples/cmdline/three_legged_dance.py index 31a22df..f0a4a20 100644 --- a/samples/cmdline/three_legged_dance.py +++ b/samples/cmdline/three_legged_dance.py @@ -1,17 +1,45 @@ -from apiclient.oauth import buzz_discovery, Flow3LO +# Copyright (C) 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -import simplejson +"""Do the OAuth 1.0a three legged dance. + +Do the OAuth 1.0a three legged dance for +a Buzz command line application. Store the generated +credentials in a common file that is used by +other example apps in the same directory. +""" + +__author__ = 'jcgregorio@google.com (Joe Gregorio)' + +from apiclient.oauth import buzz_discovery +from apiclient.oauth import FlowThreeLegged + +import pickle user_agent = 'google-api-client-python-buzz-cmdline/1.0', consumer_key = 'anonymous' consumer_secret = 'anonymous' -flow = Flow3LO(buzz_discovery, consumer_key, consumer_secret, user_agent, - domain='anonymous', - scope='https://www.googleapis.com/auth/buzz', - xoauth_displayname='Google API Client for Python Example App') +flow = FlowThreeLegged(buzz_discovery, + consumer_key='anonymous', + consumer_secret='anonymous', + user_agent='google-api-client-python-buzz-cmdline/1.0', + domain='anonymous', + scope='https://www.googleapis.com/auth/buzz', + xoauth_displayname='Google API Client Example App') -authorize_url = flow.step1() +authorize_url = flow.step1_get_authorize_url() print 'Go to the following link in your browser:' print authorize_url @@ -20,17 +48,10 @@ print accepted = 'n' while accepted.lower() == 'n': accepted = raw_input('Have you authorized me? (y/n) ') -pin = raw_input('What is the PIN? ').strip() +verification = raw_input('What is the verification code? ').strip() -access_token = flow.step2_pin(pin) - -d = dict( - consumer_key='anonymous', - consumer_secret='anonymous' - ) - -d.update(access_token) +credentials = flow.step2_exchange(verification) f = open('oauth_token.dat', 'w') -f.write(simplejson.dumps(d)) +f.write(pickle.dumps(credentials)) f.close()