Add support for Google AppEngine's urlfetch api

git-svn-id: http://freebase-python.googlecode.com/svn/trunk@41 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2

3 files changed, 183 insertions, 71 deletions

diff --git a/freebase-api/freebase/api/httplib2cookie.py b/freebase-api/freebase/api/cookie_handlers.py
index fee0ec0..88f6b4f 100755
--- a/freebase-api/freebase/api/httplib2cookie.py
+++ b/freebase-api/freebase/api/cookie_handlers.py
@@ -34,12 +34,22 @@
 #
 
 import re
-import httplib2
-from httplib2 import Http
 
-import mimetools
-import urllib
-import urllib2
+try:
+    from google.appengine.api import urlfetch
+    Http = object
+except ImportError:
+    pass
+
+try:
+    from httplib2 import Http
+except ImportError:
+    pass
+
+try:
+    import urllib
+except ImportError:
+    import urllib_stub as urllib
 import cookielib
 
 class DummyRequest(object):
@@ -84,8 +94,19 @@ class DummyRequest(object):
         #  request is the result of a redirect
         return False
 
+class DummyHttplib2Response(object):
+    """Simulated urllib2.Request object for httplib2
+
+       implements only what's necessary for cookielib.CookieJar to work
+    """
+    def __init__(self, response):
+        self.response = response
+
+    def info(self):
+        return DummyHttplib2Message(self.response)
+
 
-class DummyResponse(object):
+class DummyUrlfetchResponse(object):
     """Simulated urllib2.Request object for httplib2
 
        implements only what's necessary for cookielib.CookieJar to work
@@ -94,10 +115,10 @@ class DummyResponse(object):
         self.response = response
 
     def info(self):
-        return DummyMessage(self.response)
+        return DummyUrlfetchMessage(self.response)
 
 
-class DummyMessage(object):
+class DummyHttplib2Message(object):
     """Simulated mimetools.Message object for httplib2
 
        implements only what's necessary for cookielib.CookieJar to work
@@ -119,6 +140,28 @@ class DummyMessage(object):
         HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)')
         return [h[0] for h in HEADERVAL.findall(self.response[k])]
 
+class DummyUrlfetchMessage(object):
+    """Simulated mimetools.Message object for httplib2
+
+       implements only what's necessary for cookielib.CookieJar to work
+    """
+    def __init__(self, response):
+        self.response = response
+
+    def getheaders(self, k):
+        k = k.lower()
+        v = self.response.headers.get(k.lower(), None)
+        if k not in self.response.headers:
+            return []
+        #return self.response[k].split(re.compile(',\\s*'))
+
+        # httplib2 joins multiple values for the same header
+        #  using ','.  but the netscape cookie format uses ','
+        #  as part of the expires= date format.  so we have
+        #  to split carefully here - header.split(',') won't do it.
+        HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)')
+        return [h[0] for h in HEADERVAL.findall(self.response.headers[k])]
+
 class CookiefulHttp(Http):
     """Subclass of httplib2.Http that keeps cookie state
 
@@ -143,7 +186,30 @@ class CookiefulHttp(Http):
 
         (r, body) = Http.request(self, uri, headers=headers, **kws)
 
-        resp = DummyResponse(r)
+        resp = DummyHttplib2Response(r)
         self.cookiejar.extract_cookies(resp, req)
 
         return (r, body)
+
+class CookiefulUrlfetch(object):
+    """Class that keeps cookie state
+
+       constructor takes an optional cookiejar=cookielib.CookieJar
+    """ 
+    # TODO refactor CookefulHttp so that CookiefulUrlfetch can be a subclass of it
+    def __init__(self, cookiejar=None, **kws):
+        if cookiejar is None:
+            cookejar = cookielib.CookieJar()
+        self.cookejar = cookiejar
+
+    def request(self, uri, **kws):
+        headers = kws.pop('headers', None)
+        req = DummyRequest(uri, headers)
+        self.cookejar.add_cookie_header(req)
+        headers = req.headers
+
+        r = urlfetch.fetch(uri, headers=headers, **kws)
+
+        self.cookejar.extract_cookies(DummyUrlfetchResponse(r), req)
+        return r
+
diff --git a/freebase-api/freebase/api/httpclients.py b/freebase-api/freebase/api/httpclients.py
new file mode 100644
index 0000000..c2b10b2
--- /dev/null
+++ b/freebase-api/freebase/api/httpclients.py
@@ -0,0 +1,82 @@
+try:
+    from google.appengine.api import urlfetch
+    from cookie_handlers import CookiefulUrlfetch
+except:
+    pass
+
+try:
+    import httplib2
+    from cookie_handlers import CookiefulHttp
+except:
+    pass
+
+try:
+    import urllib2
+except:
+    pass
+
+class Urllib2Client(object):
+    def __init__(self, cookiejar):
+        cookiespy = urllib2.HTTPCookieProcessor(cookiejar)
+        self.opener = urllib2.build_opener(cookiespy)
+
+    def __call__(self, url, method, body, headers):
+        req = urllib2.Request(url, body, headers)
+
+        try:
+            resp = self.opener.open(req)
+
+        except socket.error, e:
+            self.log.error('SOCKET FAILURE: %s', e.fp.read())
+            raise MetawebError, 'failed contacting %s: %s' % (url, str(e))
+
+        except urllib2.HTTPError, e:
+            self._raise_service_error(url, e.code, e.info().type, e.fp.read())
+
+        for header in resp.info().headers:
+            self.log.debug('HTTP HEADER %s', header)
+            name, value = re.split("[:\n\r]", header, 1)
+            if name.lower() == 'x-metaweb-tid':
+                self.tid = value.strip()
+
+        return (resp, resp.read())
+
+class Httplib2Client(object):
+    def __init__(self, cookiejar):
+        self.cookiejar = cookiejar
+        self.httpclient = CookiefulHttp(cookiejar=self.cookiejar)
+
+    def __call__(self, url, method, body, headers):
+        try:
+            resp, content = self.httpclient.request(url, method=method,
+                                                    body=body, headers=headers)
+            if (resp.status != 200):
+                self._raise_service_error(url, resp.status, resp['content-type'], content)
+
+        except socket.error, e:
+            self.log.error('SOCKET FAILURE: %s', e.fp.read())
+            raise MetawebError, 'failed contacting %s: %s' % (url, str(e))
+
+        except httplib2.HttpLib2ErrorWithResponse, e:
+            self._raise_service_error(url, resp.status, resp['content-type'], content)
+        except httplib2.HttpLib2Error, e:
+            raise MetawebError(u'HTTP error: %s' % (e,))
+
+        #tid = resp.get('x-metaweb-tid', None)
+
+        return (resp, content)
+
+
+class UrlfetchClient(object):
+    def __init__(self, cookiejar):
+        self.cookiejar = cookiejar
+        self.httpclient = CookiefulUrlfetch(cookiejar=self.cookiejar)
+
+    def __call__(self, url, method, body, headers):
+        resp = self.httpclient.request(url, payload=body, method=method, headers=headers)
+
+        if resp.status_code != 200:
+            self._raise_service_error(url, resp.status_code, resp.headers['content-type'], resp.body)
+
+        return (resp, resp.content)
+
diff --git a/freebase-api/freebase/api/session.py b/freebase-api/freebase/api/session.py
index 6e3c422..73ae692 100644
--- a/freebase-api/freebase/api/session.py
+++ b/freebase-api/freebase/api/session.py
@@ -42,23 +42,38 @@ __all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict']
 __version__ = '0.1'
 
 import os, sys, re
-import urllib2
 import cookielib
-import simplejson
-from urllib import quote as urlquote
+try:
+    import simplejson
+except ImportError:
+    from django.utils import simplejson
+try:
+    from urllib import quote as urlquote
+except ImportError:
+    from urlib_stub import quote as urlquote
 import pprint
 import socket
 import logging
 
+from httpclients import Httplib2Client, Urllib2Client, UrlfetchClient
+
+# Check for urlfetch first so that urlfetch is used when running the appengine SDK
 try:
-    import httplib2
-    from httplib2cookie import CookiefulHttp
+    import google.appengine.api.urlfetch
+    from cookie_handlers import CookiefulUrlfetch
+    http_client = UrlfetchClient
 except ImportError:
-    httplib2 = None
-    CookiefulHttp = None
-    print ('freebase.api: you can install httplib2 for better performance')
+    try:
+        import httplib2
+        from cookie_handlers import CookiefulHttp
+        http_client = Httplib2Client
+    except ImportError:
+        import urllib2
+        httplib2 = None
+        CookiefulHttp = None
+        http_client = Urllib2Client
+        print ('freebase.api: you can install httplib2 for better performance')
 
-import simplejson.encoder
 # remove whitespace from json encoded output
 simplejson.JSONEncoder.item_separator = ','
 simplejson.JSONEncoder.key_separator = ':'
@@ -124,7 +139,6 @@ NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+')
 def _normalize_headers(headers):
     return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip())  for (key, value) in headers.iteritems()])
 
-
 class HTTPMetawebSession(MetawebSession):
     """
     a MetawebSession is a request/response queue.
@@ -169,11 +183,7 @@ class HTTPMetawebSession(MetawebSession):
         else:
             self.cookiejar = self._default_cookiejar
 
-        if CookiefulHttp is not None:
-            self.httpclient = CookiefulHttp(cookiejar=self.cookiejar)
-        else:
-            cookiespy = urllib2.HTTPCookieProcessor(self.cookiejar)
-            self.opener = urllib2.build_opener(cookiespy)
+        self._http_request = http_client(self.cookiejar)
 
 
     def open_cookie_file(self, cookiefile=None):
@@ -275,11 +285,7 @@ class HTTPMetawebSession(MetawebSession):
         self.log.info('%s %s%s%s', method, url, formstr, headerstr)
         #######
 
-        if CookiefulHttp is not None:
-            return self._httplib2_request(url, method, body, headers)
-        else:
-            return self._urllib2_request(url, method, body, headers)
-
+        return self._http_request(url, method, body, headers)
 
     def _raise_service_error(self, url, status, ctype, body):
 
@@ -292,48 +298,6 @@ class HTTPMetawebSession(MetawebSession):
 
         raise MetawebError, 'request failed: %s: %r %r' % (url, status, body)
         
-    def _urllib2_request(self, url, method, body, headers):
-        req = urllib2.Request(url, body, headers)
-
-        try:
-            resp = self.opener.open(req)
-
-        except socket.error, e:
-            self.log.error('SOCKET FAILURE: %s', e.fp.read())
-            raise MetawebError, 'failed contacting %s: %s' % (url, str(e))
-
-        except urllib2.HTTPError, e:
-            _raise_service_error(url, e.code, e.info().type, e.fp.read())
-
-        for header in resp.info().headers:
-            self.log.debug('HTTP HEADER %s', header)
-            name, value = re.split("[:\n\r]", header, 1)
-            if name.lower() == 'x-metaweb-tid':
-                self.tid = value.strip()
-
-        return (resp, resp.read())
-
-    def _httplib2_request(self, url, method, body, headers):
-        try:
-            resp, content = self.httpclient.request(url, method=method,
-                                                    body=body, headers=headers)
-            if (resp.status != 200):
-                self._raise_service_error(url, resp.status, resp['content-type'], content)
-
-        except socket.error, e:
-            self.log.error('SOCKET FAILURE: %s', e.fp.read())
-            raise MetawebError, 'failed contacting %s: %s' % (url, str(e))
-
-        except httplib2.HttpLib2ErrorWithResponse, e:
-            self._raise_service_error(url, resp.status, resp['content-type'], content)
-        except httplib2.HttpLib2Error, e:
-            raise MetawebError(u'HTTP error: %s' % (e,))
-
-        #tid = resp.get('x-metaweb-tid', None)
-
-        return (resp, content)
-
-
     def _httpreq_json(self, *args, **kws):
         resp, body = self._httpreq(*args, **kws)
         return self._loadjson(body)