diff options
Diffstat (limited to 'freebase-api/freebase/api')
| -rwxr-xr-x | freebase-api/freebase/api/__init__.py | 4 | ||||
| -rwxr-xr-x | freebase-api/freebase/api/cookie_handlers.py | 215 | ||||
| -rw-r--r-- | freebase-api/freebase/api/httpclients.py | 91 | ||||
| -rwxr-xr-x | freebase-api/freebase/api/mqlkey.py | 135 | ||||
| -rw-r--r-- | freebase-api/freebase/api/session.py | 687 |
5 files changed, 0 insertions, 1132 deletions
diff --git a/freebase-api/freebase/api/__init__.py b/freebase-api/freebase/api/__init__.py deleted file mode 100755 index c3adebc..0000000 --- a/freebase-api/freebase/api/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -from session import HTTPMetawebSession, MetawebError, attrdict - -from mqlkey import quotekey, unquotekey diff --git a/freebase-api/freebase/api/cookie_handlers.py b/freebase-api/freebase/api/cookie_handlers.py deleted file mode 100755 index 88f6b4f..0000000 --- a/freebase-api/freebase/api/cookie_handlers.py +++ /dev/null @@ -1,215 +0,0 @@ -# ======================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ======================================================================== - -# -# -# httplib2cookie.py allows you to use python's standard -# CookieJar class with httplib2. -# -# - -import re - -try: - from google.appengine.api import urlfetch - Http = object -except ImportError: - pass - -try: - from httplib2 import Http -except ImportError: - pass - -try: - import urllib -except ImportError: - import urllib_stub as urllib -import cookielib - -class DummyRequest(object): - """Simulated urllib2.Request object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, url, headers=None): - self.url = url - self.headers = headers - self.origin_req_host = cookielib.request_host(self) - self.type, r = urllib.splittype(url) - self.host, r = urllib.splithost(r) - if self.host: - self.host = urllib.unquote(self.host) - - def get_full_url(self): - return self.url - - def get_origin_req_host(self): - # TODO to match urllib2 this should be different for redirects - return self.origin_req_host - - def get_type(self): - return self.type - - def get_host(self): - return self.host - - def get_header(self, key, default=None): - return self.headers.get(key.lower(), default) - - def has_header(self, key): - return key in self.headers - - def add_unredirected_header(self, key, val): - # TODO this header should not be sent on redirect - self.headers[key.lower()] = val - - def is_unverifiable(self): - # TODO to match urllib2, this should be set to True when the - # request is the result of a redirect - return False - -class DummyHttplib2Response(object): - """Simulated urllib2.Request object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def info(self): - return DummyHttplib2Message(self.response) - - -class DummyUrlfetchResponse(object): - """Simulated urllib2.Request object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def info(self): - return DummyUrlfetchMessage(self.response) - - -class DummyHttplib2Message(object): - """Simulated mimetools.Message object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def getheaders(self, k): - k = k.lower() - v = self.response.get(k.lower(), None) - if k not in self.response: - return [] - #return self.response[k].split(re.compile(',\\s*')) - - # httplib2 joins multiple values for the same header - # using ','. but the netscape cookie format uses ',' - # as part of the expires= date format. so we have - # to split carefully here - header.split(',') won't do it. - HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') - return [h[0] for h in HEADERVAL.findall(self.response[k])] - -class DummyUrlfetchMessage(object): - """Simulated mimetools.Message object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def getheaders(self, k): - k = k.lower() - v = self.response.headers.get(k.lower(), None) - if k not in self.response.headers: - return [] - #return self.response[k].split(re.compile(',\\s*')) - - # httplib2 joins multiple values for the same header - # using ','. but the netscape cookie format uses ',' - # as part of the expires= date format. so we have - # to split carefully here - header.split(',') won't do it. - HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') - return [h[0] for h in HEADERVAL.findall(self.response.headers[k])] - -class CookiefulHttp(Http): - """Subclass of httplib2.Http that keeps cookie state - - constructor takes an optional cookiejar=cookielib.CookieJar - - currently this does not handle redirects completely correctly: - if the server redirects to a different host the original - cookies will still be sent to that host. - """ - def __init__(self, cookiejar=None, **kws): - # note that httplib2.Http is not a new-style-class - Http.__init__(self, **kws) - if cookiejar is None: - cookiejar = cookielib.CookieJar() - self.cookiejar = cookiejar - - def request(self, uri, **kws): - headers = kws.pop('headers', None) - req = DummyRequest(uri, headers) - self.cookiejar.add_cookie_header(req) - headers = req.headers - - (r, body) = Http.request(self, uri, headers=headers, **kws) - - resp = DummyHttplib2Response(r) - self.cookiejar.extract_cookies(resp, req) - - return (r, body) - -class CookiefulUrlfetch(object): - """Class that keeps cookie state - - constructor takes an optional cookiejar=cookielib.CookieJar - """ - # TODO refactor CookefulHttp so that CookiefulUrlfetch can be a subclass of it - def __init__(self, cookiejar=None, **kws): - if cookiejar is None: - cookejar = cookielib.CookieJar() - self.cookejar = cookiejar - - def request(self, uri, **kws): - headers = kws.pop('headers', None) - req = DummyRequest(uri, headers) - self.cookejar.add_cookie_header(req) - headers = req.headers - - r = urlfetch.fetch(uri, headers=headers, **kws) - - self.cookejar.extract_cookies(DummyUrlfetchResponse(r), req) - return r - diff --git a/freebase-api/freebase/api/httpclients.py b/freebase-api/freebase/api/httpclients.py deleted file mode 100644 index 48af9b6..0000000 --- a/freebase-api/freebase/api/httpclients.py +++ /dev/null @@ -1,91 +0,0 @@ -try: - from google.appengine.api import urlfetch - from cookie_handlers import CookiefulUrlfetch -except: - pass - -try: - import httplib2 - from cookie_handlers import CookiefulHttp -except: - pass - -try: - import urllib2 - import socket -except: - pass - -import logging -import re - -class Urllib2Client(object): - def __init__(self, cookiejar, rse): - cookiespy = urllib2.HTTPCookieProcessor(cookiejar) - self.opener = urllib2.build_opener(cookiespy) - self._raise_service_error = rse - self.log = logging.getLogger() - - def __call__(self, url, method, body, headers): - req = urllib2.Request(url, body, headers) - - try: - resp = self.opener.open(req) - - except socket.error, e: - self.log.error('SOCKET FAILURE: %s', e.fp.read()) - raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) - - except urllib2.HTTPError, e: - self.log.error('HTTP ERROR: %s', e) - self._raise_service_error(url, e.code, e.info().type, e.fp.read()) - - for header in resp.info().headers: - self.log.debug('HTTP HEADER %s', header) - name, value = re.split("[:\n\r]", header, 1) - if name.lower() == 'x-metaweb-tid': - self.tid = value.strip() - - return (resp, resp.read()) - -class Httplib2Client(object): - def __init__(self, cookiejar, rse): - self.cookiejar = cookiejar - self._raise_service_error = rse - self.httpclient = CookiefulHttp(cookiejar=self.cookiejar) - - def __call__(self, url, method, body, headers): - try: - resp, content = self.httpclient.request(url, method=method, - body=body, headers=headers) - if (resp.status != 200): - self._raise_service_error(url, resp.status, resp['content-type'], content) - - except socket.error, e: - self.log.error('SOCKET FAILURE: %s', e.fp.read()) - raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) - - except httplib2.HttpLib2ErrorWithResponse, e: - self._raise_service_error(url, resp.status, resp['content-type'], content) - except httplib2.HttpLib2Error, e: - raise MetawebError(u'HTTP error: %s' % (e,)) - - #tid = resp.get('x-metaweb-tid', None) - - return (resp, content) - - -class UrlfetchClient(object): - def __init__(self, cookiejar, rse): - self.cookiejar = cookiejar - self._raise_service_error = rse - self.httpclient = CookiefulUrlfetch(cookiejar=self.cookiejar) - - def __call__(self, url, method, body, headers): - resp = self.httpclient.request(url, payload=body, method=method, headers=headers) - - if resp.status_code != 200: - self._raise_service_error(url, resp.status_code, resp.headers['content-type'], resp.body) - - return (resp, resp.content) - diff --git a/freebase-api/freebase/api/mqlkey.py b/freebase-api/freebase/api/mqlkey.py deleted file mode 100755 index d7dd01b..0000000 --- a/freebase-api/freebase/api/mqlkey.py +++ /dev/null @@ -1,135 +0,0 @@ -# ======================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ======================================================================== - -import string -import re - -def quotekey(ustr): - """ - quote a unicode string to turn it into a valid namespace key - - """ - valid_always = string.ascii_letters + string.digits - valid_interior_only = valid_always + '_-' - - if isinstance(ustr, str): - s = unicode(ustr,'utf-8') - elif isinstance(ustr, unicode): - s = ustr - else: - raise ValueError, 'quotekey() expects utf-8 string or unicode' - - output = [] - if s[0] in valid_always: - output.append(s[0]) - else: - output.append('$%04X' % ord(s[0])) - - for c in s[1:-1]: - if c in valid_interior_only: - output.append(c) - else: - output.append('$%04X' % ord(c)) - - if len(s) > 1: - if s[-1] in valid_always: - output.append(s[-1]) - else: - output.append('$%04X' % ord(s[-1])) - - return str(''.join(output)) - - -def unquotekey(key, encoding=None): - """ - unquote a namespace key and turn it into a unicode string - """ - - valid_always = string.ascii_letters + string.digits - - output = [] - i = 0 - while i < len(key): - if key[i] in valid_always: - output.append(key[i]) - i += 1 - elif key[i] in '_-' and i != 0 and i != len(key): - output.append(key[i]) - i += 1 - elif key[i] == '$' and i+4 < len(key): - # may raise ValueError if there are invalid characters - output.append(unichr(int(key[i+1:i+5],16))) - i += 5 - else: - raise ValueError, "unquote key saw invalid character '%s' at position %d" % (key[i], i) - - ustr = u''.join(output) - - if encoding is None: - return ustr - - return ustr.encode(encoding) - - -# should this also include "'()" into safe? -def urlencode_pathseg(data): - ''' - urlencode for placement between slashes in an url. - ''' - if isinstance(data, unicode): - data = data.encode('utf_8') - return urllib.quote(data, "~:@$!*,;=&+") - - -def id_to_urlid(id): - """ - convert a mql id to an id suitable for embedding in a url path. - """ - - segs = id.split('/') - - assert isinstance(id, str) and id != '', 'bad id "%s"' % id - - if id[0] == '~': - assert len(segs) == 1 - # assume valid, should check - return id - - if id[0] == '#': - assert len(segs) == 1 - # assume valid, should check - return '%23' + id[1:] - - if id[0] != '/': - raise ValueError, 'unknown id format %s' % id - - # ok, we have a slash-path - # requote components as keys and rejoin. - # urlids do not have leading slashes!!! - return '/'.join(urlencode_pathseg(unquotekey(seg)) for seg in segs[1:]) - diff --git a/freebase-api/freebase/api/session.py b/freebase-api/freebase/api/session.py deleted file mode 100644 index 0d5159b..0000000 --- a/freebase-api/freebase/api/session.py +++ /dev/null @@ -1,687 +0,0 @@ -# ================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ==================================================================== - -""" -declarations for external metaweb api. - - - from metaweb.api import HTTPMetawebSession - - mss = HTTPMetawebSession('sandbox.freebase.com') - print mss.mqlread([dict(name=None, type='/type/type')]) -""" - - - -__all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict'] -__version__ = '0.1' - -import os, sys, re -import cookielib -try: - import simplejson -except ImportError: - try: - # appengine provides simplejson at django.utils.simplejson - from django.utils import simplejson - except ImportError: - raise Exception("unable to import simplejson") -try: - from urllib import quote as urlquote -except ImportError: - from urlib_stub import quote as urlquote -import pprint -import socket -import logging - -class Delayed(object): - """ - Wrapper for callables in log statements. Avoids actually making - the call until the result is turned into a string. - - A few examples: - - simplejson.dumps is never called because the logger never - tries to format the result - >>> logging.debug(Delayed(simplejson.dumps, q)) - - This time simplejson.dumps() is actually called: - >>> logging.warn(Delayed(simplejson.dumps, q)) - - """ - def __init__(self, f, *args, **kwds): - self.f = f - self.args = args - self.kwds = kwds - - def __str__(self): - return str(self.f(*self.args, **self.kwds)) - -def logformat(result): - """ - Format the dict/list as a json object - """ - rstr = simplejson.dumps(result, indent=2) - if rstr[0] == '{': - rstr = rstr[1:-2] - return rstr - -from httpclients import Httplib2Client, Urllib2Client, UrlfetchClient - -# Check for urlfetch first so that urlfetch is used when running the appengine SDK -try: - import google.appengine.api.urlfetch - from cookie_handlers import CookiefulUrlfetch - http_client = UrlfetchClient -except ImportError: - try: - import httplib2 - from cookie_handlers import CookiefulHttp - http_client = Httplib2Client - except ImportError: - import urllib2 - httplib2 = None - CookiefulHttp = None - http_client = Urllib2Client - -# remove whitespace from json encoded output -simplejson.JSONEncoder.item_separator = ',' -simplejson.JSONEncoder.key_separator = ':' -# don't escape slashes, we're not pasting into script tags here. -if simplejson.encoder.ESCAPE_DCT.get('/', None) == r'\/': - simplejson.encoder.ESCAPE_DCT['/'] = '/' - -def urlencode_weak(s): - return urlquote(s, safe=',/:$') - - -# from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/361668 -class attrdict(dict): - """A dict whose items can also be accessed as member variables. - - >>> d = attrdict(a=1, b=2) - >>> d['c'] = 3 - >>> print d.a, d.b, d.c - 1 2 3 - >>> d.b = 10 - >>> print d['b'] - 10 - - # but be careful, it's easy to hide methods - >>> print d.get('c') - 3 - >>> d['get'] = 4 - >>> print d.get('a') - Traceback (most recent call last): - TypeError: 'int' object is not callable - """ - def __init__(self, *args, **kwargs): - dict.__init__(self, *args, **kwargs) - self.__dict__ = self - - - -# TODO expose the common parts of the result envelope -class MetawebError(Exception): - """ - an error report from the metaweb service. - """ - pass - - - -# TODO right now this is a completely unnecessary superclass. -# is there enough common behavior between session types -# to justify it? -class MetawebSession(object): - """ - MetawebSession is the base class for MetawebSession, subclassed for - different connection types. Only http is available externally. - - This is more of an interface than a class - """ - - # interface definition here... - - -# from httplib2 -NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') -def _normalize_headers(headers): - return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) - -class HTTPMetawebSession(MetawebSession): - """ - a MetawebSession is a request/response queue. - - this version uses the HTTP api, and is synchronous. - """ - # share cookies across sessions, so that different sessions can - # see each other's writes immediately. - _default_cookiejar = cookielib.CookieJar() - - def __init__(self, service_url, username=None, password=None, prev_session=None, cookiejar=None, cookiefile=None): - """ - create a new MetawebSession for interacting with the Metaweb. - - a new session will inherit state from prev_session if present, - """ - super(HTTPMetawebSession, self).__init__() - - self.log = logging.getLogger() - - assert not service_url.endswith('/') - if not '/' in service_url: # plain host:port - service_url = 'http://' + service_url - - self.service_url = service_url - - self.username = username - self.password = password - - self.tid = None - - if prev_session: - self.service_url = prev.service_url - - if cookiefile is not None: - cookiejar = self.open_cookie_file(cookiefile) - - if cookiejar is not None: - self.cookiejar = cookiejar - elif prev_session: - self.cookiejar = prev_session.cookiejar - else: - self.cookiejar = self._default_cookiejar - - self._http_request = http_client(self.cookiejar, self._raise_service_error) - - - def open_cookie_file(self, cookiefile=None): - if cookiefile is None or cookiefile == '': - if os.environ.has_key('HOME'): - cookiefile = os.path.join(os.environ['HOME'], '.pyfreebase/cookiejar') - else: - raise MetawebError("no cookiefile specified and no $HOME/.pyfreebase directory" % cookiefile) - - cookiejar = cookielib.LWPCookieJar(cookiefile) - if os.path.exists(cookiefile): - cookiejar.load(ignore_discard=True) - - return cookiejar - - - def _httpreq(self, service_path, method='GET', body=None, form=None, - headers=None): - """ - make an http request to the service. - - form arguments are encoded in the url, even for POST, if a non-form - content-type is given for the body. - - returns a pair (resp, body) - - resp is the response object and may be different depending - on whether urllib2 or httplib2 is in use? - """ - - if method == 'GET': - assert body is None - if method != "GET" and method != "POST": - assert 0, 'unknown method %s' % method - - url = self.service_url + service_path - - if headers is None: - headers = {} - else: - headers = _normalize_headers(headers) - - # this is a lousy way to parse Content-Type, where is the library? - ct = headers.get('content-type', None) - if ct is not None: - ct = ct.split(';')[0] - - if body is not None: - # if body is provided, content-type had better be too - assert ct is not None - - if form is not None: - qstr = '&'.join(['%s=%s' % (urlencode_weak(unicode(k)), urlencode_weak(unicode(v))) - for k,v in form.items()]) - if method == 'POST': - # put the args on the url if we're putting something else - # in the body. this is used to add args to raw uploads. - if body is not None: - url += '?' + qstr - else: - if ct is None: - ct = 'application/x-www-form-urlencoded' - headers['content-type'] = ct + '; charset=utf-8' - - if ct == 'multipart/form-encoded': - # TODO handle this case - raise NotImplementedError - elif ct == 'application/x-www-form-urlencoded': - body = qstr - else: - # for all methods other than POST, use the url - url += '?' + qstr - - - # assure the service that this isn't a CSRF form submission - headers['x-metaweb-request'] = 'Python' - - if 'user-agent' not in headers: - headers['user-agent'] = 'python freebase.api-%s' % __version__ - - #if self.tid is not None: - # headers['x-metaweb-tid'] = self.tid - - ####### DEBUG MESSAGE - should check log level before generating - if form is None: - formstr = '' - else: - formstr = '\nFORM:\n ' + '\n '.join(['%s=%s' % (k,v) - for k,v in form.items()]) - if headers is None: - headerstr = '' - else: - headerstr = '\nHEADERS:\n ' + '\n '.join([('%s: %s' % (k,v)) - for k,v in headers.items()]) - self.log.info('%s %s%s%s', method, url, formstr, headerstr) - ####### - - return self._http_request(url, method, body, headers) - - def _raise_service_error(self, url, status, ctype, body): - - is_jsbody = (ctype.endswith('javascript') - or ctype.endswith('json')) - if str(status) == '400' and is_jsbody: - r = self._loadjson(body) - msg = r.messages[0] - raise MetawebError(u'%s %s %r' % (msg.get('code',''), msg.message, msg.info)) - - raise MetawebError, 'request failed: %s: %r %r' % (url, status, body) - - def _httpreq_json(self, *args, **kws): - resp, body = self._httpreq(*args, **kws) - return self._loadjson(body) - - def _loadjson(self, json): - # TODO really this should be accomplished by hooking - # simplejson to create attrdicts instead of dicts. - def struct2attrdict(st): - """ - copy a json structure, turning all dicts into attrdicts. - - copying descends instances of dict and list, including subclasses. - """ - if isinstance(st, dict): - return attrdict([(k,struct2attrdict(v)) for k,v in st.items()]) - if isinstance(st, list): - return [struct2attrdict(li) for li in st] - return st - - if json == '': - self.log.error('the empty string is not valid json') - raise MetawebError('the empty string is not valid json') - - try: - r = simplejson.loads(json) - except ValueError, e: - self.log.error('error parsing json string %r' % json) - raise MetawebError, 'error parsing JSON string: %s' % e - - return struct2attrdict(r) - - def _check_mqlerror(self, r): - if r.code != '/api/status/ok': - for msg in r.messages: - self.log.error('mql error: %s %s %r' % (msg.code, msg.message, msg.get('query', None))) - raise MetawebError, 'query failed: %s\n%r' % (r.messages[0].code, r.messages[0].get('query', None)) - - def _mqlresult(self, r): - self._check_mqlerror(r) - - self.log.info('result: %s', Delayed(logformat, r)) - - return r.result - - - - def login(self, username=None, password=None): - """sign in to the service. For a more complete description, - see http://www.freebase.com/view/en/api_account_login""" - - service = '/api/account/login' - - username = username or self.username - password = password or self.password - - assert username is not None - assert password is not None - - self.log.debug('LOGIN USERNAME: %s', username) - - r = self._httpreq_json(service, 'POST', - form=dict(username=username, - password=password)) - - if r.code != '/api/status/ok': - raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) - - self.log.debug('LOGIN RESP: %r', r) - self.log.debug('LOGIN COOKIES: %s', self.cookiejar) - - def logout(self): - """logout of the service. For a more complete description, - see http://www.freebase.com/view/en/api_account_logout""" - - service = '/api/account/logout' - - self.log.debug("LOGOUT") - - r = self._httpreq_json(service, 'GET') - - if r.code != '/api/status/ok': - raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) #this should never happen - - def user_info(self, mql_output=None): - """ get user_info. For a more complete description, - see http://www.freebase.com/view/en/api_service_user_info""" - - service = "/api/service/user_info" - - qstr = simplejson.dumps(mql_output) - - r = self._httpreq_json(service, 'POST', form=dict(mql_output=qstr)) - return r - - def loggedin(self): - """check to see whether a user is logged in or not. For a - more complete description, see http://www.freebase.com/view/en/api_account_loggedin""" - - service = "/api/account/loggedin" - try: - r = self._httpreq_json(service, 'GET') - if r.code == "/api/status/ok": - return True - - except MetawebError, me: - return False - - def mqlreaditer(self, sq, asof=None): - """read a structure query.""" - - cursor = True - - while 1: - subq = dict(query=[sq], cursor=cursor, escape=False) - if asof: - subq['as_of_time'] = asof - - qstr = simplejson.dumps(subq) - - service = '/api/service/mqlread' - - r = self._httpreq_json(service, form=dict(query=qstr)) - - for item in self._mqlresult(r): - yield item - - if r['cursor']: - cursor = r['cursor'] - self.log.info('CONTINUING with %s', cursor) - else: - return - - def mqlread(self, sq, asof=None): - """read a structure query. For a more complete description, - see http://www.freebase.com/view/en/api_service_mqlread""" - subq = dict(query=sq, escape=False) - if asof: - subq['as_of_time'] = asof - - if isinstance(sq, list): - subq['cursor'] = True - - service = '/api/service/mqlread' - - self.log.info('%s: %s', - service, - Delayed(logformat, sq)) - - qstr = simplejson.dumps(subq) - r = self._httpreq_json(service, form=dict(query=qstr)) - - return self._mqlresult(r) - - def mqlreadmulti(self, queries, asof=None): - """read a structure query""" - keys = [('q%d' % i) for i,v in enumerate(queries)]; - envelope = {} - for i,sq in enumerate(queries): - subq = dict(query=sq, escape=False) - if asof: - subq['as_of_time'] = asof - - # XXX put this back once mqlreadmulti is working in general - #if isinstance(sq, list): - # subq['cursor'] = True - envelope[keys[i]] = subq - - service = '/api/service/mqlread' - - self.log.info('%s: %s', - service, - Delayed(logformat, envelope)) - - qstr = simplejson.dumps(envelope) - rs = self._httpreq_json(service, form=dict(queries=qstr)) - - self.log.info('%s result: %s', - service, - Delayed(simplejson.dumps, rs, indent=2)) - - return [self._mqlresult(rs[key]) for key in keys] - - def raw(self, id): - """translate blob from id. For a more complete description, - see http://www.freebase.com/view/en/api_trans_raw""" - url = '/api/trans/raw' + urlquote(id) - - self.log.info(url) - - resp, body = self._httpreq(url) - - self.log.info('raw is %d bytes' % len(body)) - - return body - - def blurb(self, id, break_paragraphs=False, maxlength=200): - """translate only the text in blob from id. For a more - complete description, see http://www.freebase.com/view/en/api_trans_blurb""" - url = '/api/trans/blurb' + urlquote(id) - - self.log.info(url) - - resp, body = self._httpreq(url, form=dict(break_paragraphs=break_paragraphs, maxlength=maxlength)) - - self.log.info('blurb is %d bytes' % len(body)) - - return body - - def image_thumb(self, id, maxwidth=None, maxheight=None, mode="fit", onfail=None): - """ given the id of an image, this will return a URL of a thumbnail of the image. - The full details of how the image is cropped and finessed is detailed at - http://www.freebase.com/view/en/api_trans_image_thumb """ - - service = "/api/trans/image_thumb" - assert mode in ["fit", "fill", "fillcrop", "fillcropmid"] - - form = dict(mode=mode) - if maxwidth is not None: - form["maxwidth"] = maxwidth - if maxheight is not None: - form["maxheight"] = maxheight - if onfail is not None: - form["onfail"] = onfail - - resp, body = self._httpreq(service + urlquote(id), form=form) - self.log.info('image is %d bytes' % len(body)) - - return body - - def mqlwrite(self, sq): - """do a mql write. For a more complete description, - see http://www.freebase.com/view/en/api_service_mqlwrite""" - query = dict(query=sq, escape=False) - qstr = simplejson.dumps(query) - - self.log.debug('MQLWRITE: %s', qstr) - - service = '/api/service/mqlwrite' - - self.log.info('%s: %s', - service, - Delayed(logformat,sq)) - - r = self._httpreq_json(service, 'POST', - form=dict(query=qstr)) - - self.log.debug('MQLWRITE RESP: %r', r) - return self._mqlresult(r) - - def mqlcheck(self, sq): - """ See if a write is valid, and see what would happen, but do not - actually do the write """ - - query = dict(query=sq, escape=False) - qstr = simplejson.dumps(query) - - self.log.debug('MQLCHECK: %s', qstr) - - service = '/api/service/mqlcheck' - - self.log.info('%s: %s', - service, - Delayed(logformat, sq)) - - r = self._httpreq_json(service, 'POST', - form=dict(query=qstr)) - - - self.log.debug('MQLCHECK RESP: %r', r) - - return self._mqlresult(r) - - def mqlflush(self): - """ask the service not to hand us old data""" - self.log.debug('MQLFLUSH') - - service = '/api/service/touch' - r = self._httpreq_json(service) - - self._check_mqlerror(r) - return True - - def touch(self): - """ make sure you are accessing the most recent data. For a more - complete description, see http://www.freebase.com/view/en/api_service_touch""" - return self.mqlflush() - - - def upload(self, body, content_type, document_id=False, permission_of=False): - """upload to the metaweb. For a more complete description, - see http://www.freebase.com/view/en/api_service_upload""" - - service = '/api/service/upload' - - self.log.info('POST %s: %s (%d bytes)', - service, content_type, len(body)) - - - headers = {} - if content_type is not None: - headers['content-type'] = content_type - - form = None - if document_id is not False: - if document_id is None: - form = { 'document': '' } - else: - form = { 'document': document_id } - if permission_of is not False: - if form: - form['permission_of'] = permission_of - else: - form = { 'permission_of' : permission_of } - - # note the use of both body and form. - # form parameters get encoded into the URL in this case - r = self._httpreq_json(service, 'POST', - headers=headers, body=body, form=form) - return self._mqlresult(r) - - - def version(self): - """ get versions for various parts of freebase. For a more - complete description, see http://www.freebase.com/view/en/api_version""" - - service = "/api/version" - r = self._httpreq_json(service) - - self._check_mqlerror(r) - return r - - ### DEPRECATED IN API - def reconcile(self, name, etype=['/common/topic']): - """DEPRECATED: reconcile name to guid. For a more complete description, - see http://www.freebase.com/view/en/dataserver_reconciliation""" - - service = '/dataserver/reconciliation' - r = self._httpreq_json(service, 'GET', form={'name':name, 'types':','.join(etype)}) - - - # TODO non-conforming service, fix later - #self._mqlresult(r) - return r - - -if __name__ == '__main__': - console = logging.StreamHandler() - console.setLevel(logging.DEBUG) - - mss = HTTPMetawebSession('sandbox.freebase.com') - - self.mss.log.setLevel(logging.DEBUG) - self.mss.log.addHandler(console) - - - print mss.mqlread([dict(name=None, type='/type/type')]) |