diff options
Diffstat (limited to 'freebase-api/freebase')
| -rwxr-xr-x | freebase-api/freebase/api/cookie_handlers.py (renamed from freebase-api/freebase/api/httplib2cookie.py) | 84 | ||||
| -rw-r--r-- | freebase-api/freebase/api/httpclients.py | 82 | ||||
| -rw-r--r-- | freebase-api/freebase/api/session.py | 88 |
3 files changed, 183 insertions, 71 deletions
diff --git a/freebase-api/freebase/api/httplib2cookie.py b/freebase-api/freebase/api/cookie_handlers.py index fee0ec0..88f6b4f 100755 --- a/freebase-api/freebase/api/httplib2cookie.py +++ b/freebase-api/freebase/api/cookie_handlers.py @@ -34,12 +34,22 @@ # import re -import httplib2 -from httplib2 import Http -import mimetools -import urllib -import urllib2 +try: + from google.appengine.api import urlfetch + Http = object +except ImportError: + pass + +try: + from httplib2 import Http +except ImportError: + pass + +try: + import urllib +except ImportError: + import urllib_stub as urllib import cookielib class DummyRequest(object): @@ -84,8 +94,19 @@ class DummyRequest(object): # request is the result of a redirect return False +class DummyHttplib2Response(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def info(self): + return DummyHttplib2Message(self.response) + -class DummyResponse(object): +class DummyUrlfetchResponse(object): """Simulated urllib2.Request object for httplib2 implements only what's necessary for cookielib.CookieJar to work @@ -94,10 +115,10 @@ class DummyResponse(object): self.response = response def info(self): - return DummyMessage(self.response) + return DummyUrlfetchMessage(self.response) -class DummyMessage(object): +class DummyHttplib2Message(object): """Simulated mimetools.Message object for httplib2 implements only what's necessary for cookielib.CookieJar to work @@ -119,6 +140,28 @@ class DummyMessage(object): HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') return [h[0] for h in HEADERVAL.findall(self.response[k])] +class DummyUrlfetchMessage(object): + """Simulated mimetools.Message object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def getheaders(self, k): + k = k.lower() + v = self.response.headers.get(k.lower(), None) + if k not in self.response.headers: + return [] + #return self.response[k].split(re.compile(',\\s*')) + + # httplib2 joins multiple values for the same header + # using ','. but the netscape cookie format uses ',' + # as part of the expires= date format. so we have + # to split carefully here - header.split(',') won't do it. + HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') + return [h[0] for h in HEADERVAL.findall(self.response.headers[k])] + class CookiefulHttp(Http): """Subclass of httplib2.Http that keeps cookie state @@ -143,7 +186,30 @@ class CookiefulHttp(Http): (r, body) = Http.request(self, uri, headers=headers, **kws) - resp = DummyResponse(r) + resp = DummyHttplib2Response(r) self.cookiejar.extract_cookies(resp, req) return (r, body) + +class CookiefulUrlfetch(object): + """Class that keeps cookie state + + constructor takes an optional cookiejar=cookielib.CookieJar + """ + # TODO refactor CookefulHttp so that CookiefulUrlfetch can be a subclass of it + def __init__(self, cookiejar=None, **kws): + if cookiejar is None: + cookejar = cookielib.CookieJar() + self.cookejar = cookiejar + + def request(self, uri, **kws): + headers = kws.pop('headers', None) + req = DummyRequest(uri, headers) + self.cookejar.add_cookie_header(req) + headers = req.headers + + r = urlfetch.fetch(uri, headers=headers, **kws) + + self.cookejar.extract_cookies(DummyUrlfetchResponse(r), req) + return r + diff --git a/freebase-api/freebase/api/httpclients.py b/freebase-api/freebase/api/httpclients.py new file mode 100644 index 0000000..c2b10b2 --- /dev/null +++ b/freebase-api/freebase/api/httpclients.py @@ -0,0 +1,82 @@ +try: + from google.appengine.api import urlfetch + from cookie_handlers import CookiefulUrlfetch +except: + pass + +try: + import httplib2 + from cookie_handlers import CookiefulHttp +except: + pass + +try: + import urllib2 +except: + pass + +class Urllib2Client(object): + def __init__(self, cookiejar): + cookiespy = urllib2.HTTPCookieProcessor(cookiejar) + self.opener = urllib2.build_opener(cookiespy) + + def __call__(self, url, method, body, headers): + req = urllib2.Request(url, body, headers) + + try: + resp = self.opener.open(req) + + except socket.error, e: + self.log.error('SOCKET FAILURE: %s', e.fp.read()) + raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) + + except urllib2.HTTPError, e: + self._raise_service_error(url, e.code, e.info().type, e.fp.read()) + + for header in resp.info().headers: + self.log.debug('HTTP HEADER %s', header) + name, value = re.split("[:\n\r]", header, 1) + if name.lower() == 'x-metaweb-tid': + self.tid = value.strip() + + return (resp, resp.read()) + +class Httplib2Client(object): + def __init__(self, cookiejar): + self.cookiejar = cookiejar + self.httpclient = CookiefulHttp(cookiejar=self.cookiejar) + + def __call__(self, url, method, body, headers): + try: + resp, content = self.httpclient.request(url, method=method, + body=body, headers=headers) + if (resp.status != 200): + self._raise_service_error(url, resp.status, resp['content-type'], content) + + except socket.error, e: + self.log.error('SOCKET FAILURE: %s', e.fp.read()) + raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) + + except httplib2.HttpLib2ErrorWithResponse, e: + self._raise_service_error(url, resp.status, resp['content-type'], content) + except httplib2.HttpLib2Error, e: + raise MetawebError(u'HTTP error: %s' % (e,)) + + #tid = resp.get('x-metaweb-tid', None) + + return (resp, content) + + +class UrlfetchClient(object): + def __init__(self, cookiejar): + self.cookiejar = cookiejar + self.httpclient = CookiefulUrlfetch(cookiejar=self.cookiejar) + + def __call__(self, url, method, body, headers): + resp = self.httpclient.request(url, payload=body, method=method, headers=headers) + + if resp.status_code != 200: + self._raise_service_error(url, resp.status_code, resp.headers['content-type'], resp.body) + + return (resp, resp.content) + diff --git a/freebase-api/freebase/api/session.py b/freebase-api/freebase/api/session.py index 6e3c422..73ae692 100644 --- a/freebase-api/freebase/api/session.py +++ b/freebase-api/freebase/api/session.py @@ -42,23 +42,38 @@ __all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict'] __version__ = '0.1' import os, sys, re -import urllib2 import cookielib -import simplejson -from urllib import quote as urlquote +try: + import simplejson +except ImportError: + from django.utils import simplejson +try: + from urllib import quote as urlquote +except ImportError: + from urlib_stub import quote as urlquote import pprint import socket import logging +from httpclients import Httplib2Client, Urllib2Client, UrlfetchClient + +# Check for urlfetch first so that urlfetch is used when running the appengine SDK try: - import httplib2 - from httplib2cookie import CookiefulHttp + import google.appengine.api.urlfetch + from cookie_handlers import CookiefulUrlfetch + http_client = UrlfetchClient except ImportError: - httplib2 = None - CookiefulHttp = None - print ('freebase.api: you can install httplib2 for better performance') + try: + import httplib2 + from cookie_handlers import CookiefulHttp + http_client = Httplib2Client + except ImportError: + import urllib2 + httplib2 = None + CookiefulHttp = None + http_client = Urllib2Client + print ('freebase.api: you can install httplib2 for better performance') -import simplejson.encoder # remove whitespace from json encoded output simplejson.JSONEncoder.item_separator = ',' simplejson.JSONEncoder.key_separator = ':' @@ -124,7 +139,6 @@ NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') def _normalize_headers(headers): return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) - class HTTPMetawebSession(MetawebSession): """ a MetawebSession is a request/response queue. @@ -169,11 +183,7 @@ class HTTPMetawebSession(MetawebSession): else: self.cookiejar = self._default_cookiejar - if CookiefulHttp is not None: - self.httpclient = CookiefulHttp(cookiejar=self.cookiejar) - else: - cookiespy = urllib2.HTTPCookieProcessor(self.cookiejar) - self.opener = urllib2.build_opener(cookiespy) + self._http_request = http_client(self.cookiejar) def open_cookie_file(self, cookiefile=None): @@ -275,11 +285,7 @@ class HTTPMetawebSession(MetawebSession): self.log.info('%s %s%s%s', method, url, formstr, headerstr) ####### - if CookiefulHttp is not None: - return self._httplib2_request(url, method, body, headers) - else: - return self._urllib2_request(url, method, body, headers) - + return self._http_request(url, method, body, headers) def _raise_service_error(self, url, status, ctype, body): @@ -292,48 +298,6 @@ class HTTPMetawebSession(MetawebSession): raise MetawebError, 'request failed: %s: %r %r' % (url, status, body) - def _urllib2_request(self, url, method, body, headers): - req = urllib2.Request(url, body, headers) - - try: - resp = self.opener.open(req) - - except socket.error, e: - self.log.error('SOCKET FAILURE: %s', e.fp.read()) - raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) - - except urllib2.HTTPError, e: - _raise_service_error(url, e.code, e.info().type, e.fp.read()) - - for header in resp.info().headers: - self.log.debug('HTTP HEADER %s', header) - name, value = re.split("[:\n\r]", header, 1) - if name.lower() == 'x-metaweb-tid': - self.tid = value.strip() - - return (resp, resp.read()) - - def _httplib2_request(self, url, method, body, headers): - try: - resp, content = self.httpclient.request(url, method=method, - body=body, headers=headers) - if (resp.status != 200): - self._raise_service_error(url, resp.status, resp['content-type'], content) - - except socket.error, e: - self.log.error('SOCKET FAILURE: %s', e.fp.read()) - raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) - - except httplib2.HttpLib2ErrorWithResponse, e: - self._raise_service_error(url, resp.status, resp['content-type'], content) - except httplib2.HttpLib2Error, e: - raise MetawebError(u'HTTP error: %s' % (e,)) - - #tid = resp.get('x-metaweb-tid', None) - - return (resp, content) - - def _httpreq_json(self, *args, **kws): resp, body = self._httpreq(*args, **kws) return self._loadjson(body) |