From aad119a9686ec04abec7c639976dd4235d93ec56 Mon Sep 17 00:00:00 2001 From: "alex.boterolowry" Date: Fri, 13 Jun 2008 18:47:48 +0000 Subject: Add support for Google AppEngine's urlfetch api git-svn-id: http://freebase-python.googlecode.com/svn/trunk@41 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2 --- freebase-api/freebase/api/cookie_handlers.py | 215 +++++++++++++++++++++++++++ freebase-api/freebase/api/httpclients.py | 82 ++++++++++ freebase-api/freebase/api/httplib2cookie.py | 149 ------------------- freebase-api/freebase/api/session.py | 88 ++++------- 4 files changed, 323 insertions(+), 211 deletions(-) create mode 100755 freebase-api/freebase/api/cookie_handlers.py create mode 100644 freebase-api/freebase/api/httpclients.py delete mode 100755 freebase-api/freebase/api/httplib2cookie.py (limited to 'freebase-api/freebase') diff --git a/freebase-api/freebase/api/cookie_handlers.py b/freebase-api/freebase/api/cookie_handlers.py new file mode 100755 index 0000000..88f6b4f --- /dev/null +++ b/freebase-api/freebase/api/cookie_handlers.py @@ -0,0 +1,215 @@ +# ======================================================================== +# Copyright (c) 2007, Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ======================================================================== + +# +# +# httplib2cookie.py allows you to use python's standard +# CookieJar class with httplib2. +# +# + +import re + +try: + from google.appengine.api import urlfetch + Http = object +except ImportError: + pass + +try: + from httplib2 import Http +except ImportError: + pass + +try: + import urllib +except ImportError: + import urllib_stub as urllib +import cookielib + +class DummyRequest(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, url, headers=None): + self.url = url + self.headers = headers + self.origin_req_host = cookielib.request_host(self) + self.type, r = urllib.splittype(url) + self.host, r = urllib.splithost(r) + if self.host: + self.host = urllib.unquote(self.host) + + def get_full_url(self): + return self.url + + def get_origin_req_host(self): + # TODO to match urllib2 this should be different for redirects + return self.origin_req_host + + def get_type(self): + return self.type + + def get_host(self): + return self.host + + def get_header(self, key, default=None): + return self.headers.get(key.lower(), default) + + def has_header(self, key): + return key in self.headers + + def add_unredirected_header(self, key, val): + # TODO this header should not be sent on redirect + self.headers[key.lower()] = val + + def is_unverifiable(self): + # TODO to match urllib2, this should be set to True when the + # request is the result of a redirect + return False + +class DummyHttplib2Response(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def info(self): + return DummyHttplib2Message(self.response) + + +class DummyUrlfetchResponse(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def info(self): + return DummyUrlfetchMessage(self.response) + + +class DummyHttplib2Message(object): + """Simulated mimetools.Message object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def getheaders(self, k): + k = k.lower() + v = self.response.get(k.lower(), None) + if k not in self.response: + return [] + #return self.response[k].split(re.compile(',\\s*')) + + # httplib2 joins multiple values for the same header + # using ','. but the netscape cookie format uses ',' + # as part of the expires= date format. so we have + # to split carefully here - header.split(',') won't do it. + HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') + return [h[0] for h in HEADERVAL.findall(self.response[k])] + +class DummyUrlfetchMessage(object): + """Simulated mimetools.Message object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def getheaders(self, k): + k = k.lower() + v = self.response.headers.get(k.lower(), None) + if k not in self.response.headers: + return [] + #return self.response[k].split(re.compile(',\\s*')) + + # httplib2 joins multiple values for the same header + # using ','. but the netscape cookie format uses ',' + # as part of the expires= date format. so we have + # to split carefully here - header.split(',') won't do it. + HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') + return [h[0] for h in HEADERVAL.findall(self.response.headers[k])] + +class CookiefulHttp(Http): + """Subclass of httplib2.Http that keeps cookie state + + constructor takes an optional cookiejar=cookielib.CookieJar + + currently this does not handle redirects completely correctly: + if the server redirects to a different host the original + cookies will still be sent to that host. + """ + def __init__(self, cookiejar=None, **kws): + # note that httplib2.Http is not a new-style-class + Http.__init__(self, **kws) + if cookiejar is None: + cookiejar = cookielib.CookieJar() + self.cookiejar = cookiejar + + def request(self, uri, **kws): + headers = kws.pop('headers', None) + req = DummyRequest(uri, headers) + self.cookiejar.add_cookie_header(req) + headers = req.headers + + (r, body) = Http.request(self, uri, headers=headers, **kws) + + resp = DummyHttplib2Response(r) + self.cookiejar.extract_cookies(resp, req) + + return (r, body) + +class CookiefulUrlfetch(object): + """Class that keeps cookie state + + constructor takes an optional cookiejar=cookielib.CookieJar + """ + # TODO refactor CookefulHttp so that CookiefulUrlfetch can be a subclass of it + def __init__(self, cookiejar=None, **kws): + if cookiejar is None: + cookejar = cookielib.CookieJar() + self.cookejar = cookiejar + + def request(self, uri, **kws): + headers = kws.pop('headers', None) + req = DummyRequest(uri, headers) + self.cookejar.add_cookie_header(req) + headers = req.headers + + r = urlfetch.fetch(uri, headers=headers, **kws) + + self.cookejar.extract_cookies(DummyUrlfetchResponse(r), req) + return r + diff --git a/freebase-api/freebase/api/httpclients.py b/freebase-api/freebase/api/httpclients.py new file mode 100644 index 0000000..c2b10b2 --- /dev/null +++ b/freebase-api/freebase/api/httpclients.py @@ -0,0 +1,82 @@ +try: + from google.appengine.api import urlfetch + from cookie_handlers import CookiefulUrlfetch +except: + pass + +try: + import httplib2 + from cookie_handlers import CookiefulHttp +except: + pass + +try: + import urllib2 +except: + pass + +class Urllib2Client(object): + def __init__(self, cookiejar): + cookiespy = urllib2.HTTPCookieProcessor(cookiejar) + self.opener = urllib2.build_opener(cookiespy) + + def __call__(self, url, method, body, headers): + req = urllib2.Request(url, body, headers) + + try: + resp = self.opener.open(req) + + except socket.error, e: + self.log.error('SOCKET FAILURE: %s', e.fp.read()) + raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) + + except urllib2.HTTPError, e: + self._raise_service_error(url, e.code, e.info().type, e.fp.read()) + + for header in resp.info().headers: + self.log.debug('HTTP HEADER %s', header) + name, value = re.split("[:\n\r]", header, 1) + if name.lower() == 'x-metaweb-tid': + self.tid = value.strip() + + return (resp, resp.read()) + +class Httplib2Client(object): + def __init__(self, cookiejar): + self.cookiejar = cookiejar + self.httpclient = CookiefulHttp(cookiejar=self.cookiejar) + + def __call__(self, url, method, body, headers): + try: + resp, content = self.httpclient.request(url, method=method, + body=body, headers=headers) + if (resp.status != 200): + self._raise_service_error(url, resp.status, resp['content-type'], content) + + except socket.error, e: + self.log.error('SOCKET FAILURE: %s', e.fp.read()) + raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) + + except httplib2.HttpLib2ErrorWithResponse, e: + self._raise_service_error(url, resp.status, resp['content-type'], content) + except httplib2.HttpLib2Error, e: + raise MetawebError(u'HTTP error: %s' % (e,)) + + #tid = resp.get('x-metaweb-tid', None) + + return (resp, content) + + +class UrlfetchClient(object): + def __init__(self, cookiejar): + self.cookiejar = cookiejar + self.httpclient = CookiefulUrlfetch(cookiejar=self.cookiejar) + + def __call__(self, url, method, body, headers): + resp = self.httpclient.request(url, payload=body, method=method, headers=headers) + + if resp.status_code != 200: + self._raise_service_error(url, resp.status_code, resp.headers['content-type'], resp.body) + + return (resp, resp.content) + diff --git a/freebase-api/freebase/api/httplib2cookie.py b/freebase-api/freebase/api/httplib2cookie.py deleted file mode 100755 index fee0ec0..0000000 --- a/freebase-api/freebase/api/httplib2cookie.py +++ /dev/null @@ -1,149 +0,0 @@ -# ======================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ======================================================================== - -# -# -# httplib2cookie.py allows you to use python's standard -# CookieJar class with httplib2. -# -# - -import re -import httplib2 -from httplib2 import Http - -import mimetools -import urllib -import urllib2 -import cookielib - -class DummyRequest(object): - """Simulated urllib2.Request object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, url, headers=None): - self.url = url - self.headers = headers - self.origin_req_host = cookielib.request_host(self) - self.type, r = urllib.splittype(url) - self.host, r = urllib.splithost(r) - if self.host: - self.host = urllib.unquote(self.host) - - def get_full_url(self): - return self.url - - def get_origin_req_host(self): - # TODO to match urllib2 this should be different for redirects - return self.origin_req_host - - def get_type(self): - return self.type - - def get_host(self): - return self.host - - def get_header(self, key, default=None): - return self.headers.get(key.lower(), default) - - def has_header(self, key): - return key in self.headers - - def add_unredirected_header(self, key, val): - # TODO this header should not be sent on redirect - self.headers[key.lower()] = val - - def is_unverifiable(self): - # TODO to match urllib2, this should be set to True when the - # request is the result of a redirect - return False - - -class DummyResponse(object): - """Simulated urllib2.Request object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def info(self): - return DummyMessage(self.response) - - -class DummyMessage(object): - """Simulated mimetools.Message object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def getheaders(self, k): - k = k.lower() - v = self.response.get(k.lower(), None) - if k not in self.response: - return [] - #return self.response[k].split(re.compile(',\\s*')) - - # httplib2 joins multiple values for the same header - # using ','. but the netscape cookie format uses ',' - # as part of the expires= date format. so we have - # to split carefully here - header.split(',') won't do it. - HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') - return [h[0] for h in HEADERVAL.findall(self.response[k])] - -class CookiefulHttp(Http): - """Subclass of httplib2.Http that keeps cookie state - - constructor takes an optional cookiejar=cookielib.CookieJar - - currently this does not handle redirects completely correctly: - if the server redirects to a different host the original - cookies will still be sent to that host. - """ - def __init__(self, cookiejar=None, **kws): - # note that httplib2.Http is not a new-style-class - Http.__init__(self, **kws) - if cookiejar is None: - cookiejar = cookielib.CookieJar() - self.cookiejar = cookiejar - - def request(self, uri, **kws): - headers = kws.pop('headers', None) - req = DummyRequest(uri, headers) - self.cookiejar.add_cookie_header(req) - headers = req.headers - - (r, body) = Http.request(self, uri, headers=headers, **kws) - - resp = DummyResponse(r) - self.cookiejar.extract_cookies(resp, req) - - return (r, body) diff --git a/freebase-api/freebase/api/session.py b/freebase-api/freebase/api/session.py index 6e3c422..73ae692 100644 --- a/freebase-api/freebase/api/session.py +++ b/freebase-api/freebase/api/session.py @@ -42,23 +42,38 @@ __all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict'] __version__ = '0.1' import os, sys, re -import urllib2 import cookielib -import simplejson -from urllib import quote as urlquote +try: + import simplejson +except ImportError: + from django.utils import simplejson +try: + from urllib import quote as urlquote +except ImportError: + from urlib_stub import quote as urlquote import pprint import socket import logging +from httpclients import Httplib2Client, Urllib2Client, UrlfetchClient + +# Check for urlfetch first so that urlfetch is used when running the appengine SDK try: - import httplib2 - from httplib2cookie import CookiefulHttp + import google.appengine.api.urlfetch + from cookie_handlers import CookiefulUrlfetch + http_client = UrlfetchClient except ImportError: - httplib2 = None - CookiefulHttp = None - print ('freebase.api: you can install httplib2 for better performance') + try: + import httplib2 + from cookie_handlers import CookiefulHttp + http_client = Httplib2Client + except ImportError: + import urllib2 + httplib2 = None + CookiefulHttp = None + http_client = Urllib2Client + print ('freebase.api: you can install httplib2 for better performance') -import simplejson.encoder # remove whitespace from json encoded output simplejson.JSONEncoder.item_separator = ',' simplejson.JSONEncoder.key_separator = ':' @@ -124,7 +139,6 @@ NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') def _normalize_headers(headers): return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) - class HTTPMetawebSession(MetawebSession): """ a MetawebSession is a request/response queue. @@ -169,11 +183,7 @@ class HTTPMetawebSession(MetawebSession): else: self.cookiejar = self._default_cookiejar - if CookiefulHttp is not None: - self.httpclient = CookiefulHttp(cookiejar=self.cookiejar) - else: - cookiespy = urllib2.HTTPCookieProcessor(self.cookiejar) - self.opener = urllib2.build_opener(cookiespy) + self._http_request = http_client(self.cookiejar) def open_cookie_file(self, cookiefile=None): @@ -275,11 +285,7 @@ class HTTPMetawebSession(MetawebSession): self.log.info('%s %s%s%s', method, url, formstr, headerstr) ####### - if CookiefulHttp is not None: - return self._httplib2_request(url, method, body, headers) - else: - return self._urllib2_request(url, method, body, headers) - + return self._http_request(url, method, body, headers) def _raise_service_error(self, url, status, ctype, body): @@ -292,48 +298,6 @@ class HTTPMetawebSession(MetawebSession): raise MetawebError, 'request failed: %s: %r %r' % (url, status, body) - def _urllib2_request(self, url, method, body, headers): - req = urllib2.Request(url, body, headers) - - try: - resp = self.opener.open(req) - - except socket.error, e: - self.log.error('SOCKET FAILURE: %s', e.fp.read()) - raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) - - except urllib2.HTTPError, e: - _raise_service_error(url, e.code, e.info().type, e.fp.read()) - - for header in resp.info().headers: - self.log.debug('HTTP HEADER %s', header) - name, value = re.split("[:\n\r]", header, 1) - if name.lower() == 'x-metaweb-tid': - self.tid = value.strip() - - return (resp, resp.read()) - - def _httplib2_request(self, url, method, body, headers): - try: - resp, content = self.httpclient.request(url, method=method, - body=body, headers=headers) - if (resp.status != 200): - self._raise_service_error(url, resp.status, resp['content-type'], content) - - except socket.error, e: - self.log.error('SOCKET FAILURE: %s', e.fp.read()) - raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) - - except httplib2.HttpLib2ErrorWithResponse, e: - self._raise_service_error(url, resp.status, resp['content-type'], content) - except httplib2.HttpLib2Error, e: - raise MetawebError(u'HTTP error: %s' % (e,)) - - #tid = resp.get('x-metaweb-tid', None) - - return (resp, content) - - def _httpreq_json(self, *args, **kws): resp, body = self._httpreq(*args, **kws) return self._loadjson(body) -- cgit v1.3.1