From dd2da52625e890e29a6fdbba43456dd8475d51ac Mon Sep 17 00:00:00 2001 From: nitromaster101 Date: Wed, 17 Jun 2009 19:02:50 +0000 Subject: reorganize package inorder to emphasize freebase-api as the main one git-svn-id: http://freebase-python.googlecode.com/svn/trunk@88 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2 --- freebase/api/__init__.py | 4 + freebase/api/cookie_handlers.py | 215 +++++++++++++ freebase/api/httpclients.py | 91 ++++++ freebase/api/mqlkey.py | 135 ++++++++ freebase/api/session.py | 687 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 1132 insertions(+) create mode 100755 freebase/api/__init__.py create mode 100755 freebase/api/cookie_handlers.py create mode 100644 freebase/api/httpclients.py create mode 100755 freebase/api/mqlkey.py create mode 100644 freebase/api/session.py (limited to 'freebase/api') diff --git a/freebase/api/__init__.py b/freebase/api/__init__.py new file mode 100755 index 0000000..c3adebc --- /dev/null +++ b/freebase/api/__init__.py @@ -0,0 +1,4 @@ + +from session import HTTPMetawebSession, MetawebError, attrdict + +from mqlkey import quotekey, unquotekey diff --git a/freebase/api/cookie_handlers.py b/freebase/api/cookie_handlers.py new file mode 100755 index 0000000..88f6b4f --- /dev/null +++ b/freebase/api/cookie_handlers.py @@ -0,0 +1,215 @@ +# ======================================================================== +# Copyright (c) 2007, Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ======================================================================== + +# +# +# httplib2cookie.py allows you to use python's standard +# CookieJar class with httplib2. +# +# + +import re + +try: + from google.appengine.api import urlfetch + Http = object +except ImportError: + pass + +try: + from httplib2 import Http +except ImportError: + pass + +try: + import urllib +except ImportError: + import urllib_stub as urllib +import cookielib + +class DummyRequest(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, url, headers=None): + self.url = url + self.headers = headers + self.origin_req_host = cookielib.request_host(self) + self.type, r = urllib.splittype(url) + self.host, r = urllib.splithost(r) + if self.host: + self.host = urllib.unquote(self.host) + + def get_full_url(self): + return self.url + + def get_origin_req_host(self): + # TODO to match urllib2 this should be different for redirects + return self.origin_req_host + + def get_type(self): + return self.type + + def get_host(self): + return self.host + + def get_header(self, key, default=None): + return self.headers.get(key.lower(), default) + + def has_header(self, key): + return key in self.headers + + def add_unredirected_header(self, key, val): + # TODO this header should not be sent on redirect + self.headers[key.lower()] = val + + def is_unverifiable(self): + # TODO to match urllib2, this should be set to True when the + # request is the result of a redirect + return False + +class DummyHttplib2Response(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def info(self): + return DummyHttplib2Message(self.response) + + +class DummyUrlfetchResponse(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def info(self): + return DummyUrlfetchMessage(self.response) + + +class DummyHttplib2Message(object): + """Simulated mimetools.Message object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def getheaders(self, k): + k = k.lower() + v = self.response.get(k.lower(), None) + if k not in self.response: + return [] + #return self.response[k].split(re.compile(',\\s*')) + + # httplib2 joins multiple values for the same header + # using ','. but the netscape cookie format uses ',' + # as part of the expires= date format. so we have + # to split carefully here - header.split(',') won't do it. + HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') + return [h[0] for h in HEADERVAL.findall(self.response[k])] + +class DummyUrlfetchMessage(object): + """Simulated mimetools.Message object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def getheaders(self, k): + k = k.lower() + v = self.response.headers.get(k.lower(), None) + if k not in self.response.headers: + return [] + #return self.response[k].split(re.compile(',\\s*')) + + # httplib2 joins multiple values for the same header + # using ','. but the netscape cookie format uses ',' + # as part of the expires= date format. so we have + # to split carefully here - header.split(',') won't do it. + HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') + return [h[0] for h in HEADERVAL.findall(self.response.headers[k])] + +class CookiefulHttp(Http): + """Subclass of httplib2.Http that keeps cookie state + + constructor takes an optional cookiejar=cookielib.CookieJar + + currently this does not handle redirects completely correctly: + if the server redirects to a different host the original + cookies will still be sent to that host. + """ + def __init__(self, cookiejar=None, **kws): + # note that httplib2.Http is not a new-style-class + Http.__init__(self, **kws) + if cookiejar is None: + cookiejar = cookielib.CookieJar() + self.cookiejar = cookiejar + + def request(self, uri, **kws): + headers = kws.pop('headers', None) + req = DummyRequest(uri, headers) + self.cookiejar.add_cookie_header(req) + headers = req.headers + + (r, body) = Http.request(self, uri, headers=headers, **kws) + + resp = DummyHttplib2Response(r) + self.cookiejar.extract_cookies(resp, req) + + return (r, body) + +class CookiefulUrlfetch(object): + """Class that keeps cookie state + + constructor takes an optional cookiejar=cookielib.CookieJar + """ + # TODO refactor CookefulHttp so that CookiefulUrlfetch can be a subclass of it + def __init__(self, cookiejar=None, **kws): + if cookiejar is None: + cookejar = cookielib.CookieJar() + self.cookejar = cookiejar + + def request(self, uri, **kws): + headers = kws.pop('headers', None) + req = DummyRequest(uri, headers) + self.cookejar.add_cookie_header(req) + headers = req.headers + + r = urlfetch.fetch(uri, headers=headers, **kws) + + self.cookejar.extract_cookies(DummyUrlfetchResponse(r), req) + return r + diff --git a/freebase/api/httpclients.py b/freebase/api/httpclients.py new file mode 100644 index 0000000..48af9b6 --- /dev/null +++ b/freebase/api/httpclients.py @@ -0,0 +1,91 @@ +try: + from google.appengine.api import urlfetch + from cookie_handlers import CookiefulUrlfetch +except: + pass + +try: + import httplib2 + from cookie_handlers import CookiefulHttp +except: + pass + +try: + import urllib2 + import socket +except: + pass + +import logging +import re + +class Urllib2Client(object): + def __init__(self, cookiejar, rse): + cookiespy = urllib2.HTTPCookieProcessor(cookiejar) + self.opener = urllib2.build_opener(cookiespy) + self._raise_service_error = rse + self.log = logging.getLogger() + + def __call__(self, url, method, body, headers): + req = urllib2.Request(url, body, headers) + + try: + resp = self.opener.open(req) + + except socket.error, e: + self.log.error('SOCKET FAILURE: %s', e.fp.read()) + raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) + + except urllib2.HTTPError, e: + self.log.error('HTTP ERROR: %s', e) + self._raise_service_error(url, e.code, e.info().type, e.fp.read()) + + for header in resp.info().headers: + self.log.debug('HTTP HEADER %s', header) + name, value = re.split("[:\n\r]", header, 1) + if name.lower() == 'x-metaweb-tid': + self.tid = value.strip() + + return (resp, resp.read()) + +class Httplib2Client(object): + def __init__(self, cookiejar, rse): + self.cookiejar = cookiejar + self._raise_service_error = rse + self.httpclient = CookiefulHttp(cookiejar=self.cookiejar) + + def __call__(self, url, method, body, headers): + try: + resp, content = self.httpclient.request(url, method=method, + body=body, headers=headers) + if (resp.status != 200): + self._raise_service_error(url, resp.status, resp['content-type'], content) + + except socket.error, e: + self.log.error('SOCKET FAILURE: %s', e.fp.read()) + raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) + + except httplib2.HttpLib2ErrorWithResponse, e: + self._raise_service_error(url, resp.status, resp['content-type'], content) + except httplib2.HttpLib2Error, e: + raise MetawebError(u'HTTP error: %s' % (e,)) + + #tid = resp.get('x-metaweb-tid', None) + + return (resp, content) + + +class UrlfetchClient(object): + def __init__(self, cookiejar, rse): + self.cookiejar = cookiejar + self._raise_service_error = rse + self.httpclient = CookiefulUrlfetch(cookiejar=self.cookiejar) + + def __call__(self, url, method, body, headers): + resp = self.httpclient.request(url, payload=body, method=method, headers=headers) + + if resp.status_code != 200: + self._raise_service_error(url, resp.status_code, resp.headers['content-type'], resp.body) + + return (resp, resp.content) + diff --git a/freebase/api/mqlkey.py b/freebase/api/mqlkey.py new file mode 100755 index 0000000..d7dd01b --- /dev/null +++ b/freebase/api/mqlkey.py @@ -0,0 +1,135 @@ +# ======================================================================== +# Copyright (c) 2007, Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ======================================================================== + +import string +import re + +def quotekey(ustr): + """ + quote a unicode string to turn it into a valid namespace key + + """ + valid_always = string.ascii_letters + string.digits + valid_interior_only = valid_always + '_-' + + if isinstance(ustr, str): + s = unicode(ustr,'utf-8') + elif isinstance(ustr, unicode): + s = ustr + else: + raise ValueError, 'quotekey() expects utf-8 string or unicode' + + output = [] + if s[0] in valid_always: + output.append(s[0]) + else: + output.append('$%04X' % ord(s[0])) + + for c in s[1:-1]: + if c in valid_interior_only: + output.append(c) + else: + output.append('$%04X' % ord(c)) + + if len(s) > 1: + if s[-1] in valid_always: + output.append(s[-1]) + else: + output.append('$%04X' % ord(s[-1])) + + return str(''.join(output)) + + +def unquotekey(key, encoding=None): + """ + unquote a namespace key and turn it into a unicode string + """ + + valid_always = string.ascii_letters + string.digits + + output = [] + i = 0 + while i < len(key): + if key[i] in valid_always: + output.append(key[i]) + i += 1 + elif key[i] in '_-' and i != 0 and i != len(key): + output.append(key[i]) + i += 1 + elif key[i] == '$' and i+4 < len(key): + # may raise ValueError if there are invalid characters + output.append(unichr(int(key[i+1:i+5],16))) + i += 5 + else: + raise ValueError, "unquote key saw invalid character '%s' at position %d" % (key[i], i) + + ustr = u''.join(output) + + if encoding is None: + return ustr + + return ustr.encode(encoding) + + +# should this also include "'()" into safe? +def urlencode_pathseg(data): + ''' + urlencode for placement between slashes in an url. + ''' + if isinstance(data, unicode): + data = data.encode('utf_8') + return urllib.quote(data, "~:@$!*,;=&+") + + +def id_to_urlid(id): + """ + convert a mql id to an id suitable for embedding in a url path. + """ + + segs = id.split('/') + + assert isinstance(id, str) and id != '', 'bad id "%s"' % id + + if id[0] == '~': + assert len(segs) == 1 + # assume valid, should check + return id + + if id[0] == '#': + assert len(segs) == 1 + # assume valid, should check + return '%23' + id[1:] + + if id[0] != '/': + raise ValueError, 'unknown id format %s' % id + + # ok, we have a slash-path + # requote components as keys and rejoin. + # urlids do not have leading slashes!!! + return '/'.join(urlencode_pathseg(unquotekey(seg)) for seg in segs[1:]) + diff --git a/freebase/api/session.py b/freebase/api/session.py new file mode 100644 index 0000000..0d5159b --- /dev/null +++ b/freebase/api/session.py @@ -0,0 +1,687 @@ +# ================================================================== +# Copyright (c) 2007, Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + +""" +declarations for external metaweb api. + + + from metaweb.api import HTTPMetawebSession + + mss = HTTPMetawebSession('sandbox.freebase.com') + print mss.mqlread([dict(name=None, type='/type/type')]) +""" + + + +__all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict'] +__version__ = '0.1' + +import os, sys, re +import cookielib +try: + import simplejson +except ImportError: + try: + # appengine provides simplejson at django.utils.simplejson + from django.utils import simplejson + except ImportError: + raise Exception("unable to import simplejson") +try: + from urllib import quote as urlquote +except ImportError: + from urlib_stub import quote as urlquote +import pprint +import socket +import logging + +class Delayed(object): + """ + Wrapper for callables in log statements. Avoids actually making + the call until the result is turned into a string. + + A few examples: + + simplejson.dumps is never called because the logger never + tries to format the result + >>> logging.debug(Delayed(simplejson.dumps, q)) + + This time simplejson.dumps() is actually called: + >>> logging.warn(Delayed(simplejson.dumps, q)) + + """ + def __init__(self, f, *args, **kwds): + self.f = f + self.args = args + self.kwds = kwds + + def __str__(self): + return str(self.f(*self.args, **self.kwds)) + +def logformat(result): + """ + Format the dict/list as a json object + """ + rstr = simplejson.dumps(result, indent=2) + if rstr[0] == '{': + rstr = rstr[1:-2] + return rstr + +from httpclients import Httplib2Client, Urllib2Client, UrlfetchClient + +# Check for urlfetch first so that urlfetch is used when running the appengine SDK +try: + import google.appengine.api.urlfetch + from cookie_handlers import CookiefulUrlfetch + http_client = UrlfetchClient +except ImportError: + try: + import httplib2 + from cookie_handlers import CookiefulHttp + http_client = Httplib2Client + except ImportError: + import urllib2 + httplib2 = None + CookiefulHttp = None + http_client = Urllib2Client + +# remove whitespace from json encoded output +simplejson.JSONEncoder.item_separator = ',' +simplejson.JSONEncoder.key_separator = ':' +# don't escape slashes, we're not pasting into script tags here. +if simplejson.encoder.ESCAPE_DCT.get('/', None) == r'\/': + simplejson.encoder.ESCAPE_DCT['/'] = '/' + +def urlencode_weak(s): + return urlquote(s, safe=',/:$') + + +# from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/361668 +class attrdict(dict): + """A dict whose items can also be accessed as member variables. + + >>> d = attrdict(a=1, b=2) + >>> d['c'] = 3 + >>> print d.a, d.b, d.c + 1 2 3 + >>> d.b = 10 + >>> print d['b'] + 10 + + # but be careful, it's easy to hide methods + >>> print d.get('c') + 3 + >>> d['get'] = 4 + >>> print d.get('a') + Traceback (most recent call last): + TypeError: 'int' object is not callable + """ + def __init__(self, *args, **kwargs): + dict.__init__(self, *args, **kwargs) + self.__dict__ = self + + + +# TODO expose the common parts of the result envelope +class MetawebError(Exception): + """ + an error report from the metaweb service. + """ + pass + + + +# TODO right now this is a completely unnecessary superclass. +# is there enough common behavior between session types +# to justify it? +class MetawebSession(object): + """ + MetawebSession is the base class for MetawebSession, subclassed for + different connection types. Only http is available externally. + + This is more of an interface than a class + """ + + # interface definition here... + + +# from httplib2 +NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') +def _normalize_headers(headers): + return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) + +class HTTPMetawebSession(MetawebSession): + """ + a MetawebSession is a request/response queue. + + this version uses the HTTP api, and is synchronous. + """ + # share cookies across sessions, so that different sessions can + # see each other's writes immediately. + _default_cookiejar = cookielib.CookieJar() + + def __init__(self, service_url, username=None, password=None, prev_session=None, cookiejar=None, cookiefile=None): + """ + create a new MetawebSession for interacting with the Metaweb. + + a new session will inherit state from prev_session if present, + """ + super(HTTPMetawebSession, self).__init__() + + self.log = logging.getLogger() + + assert not service_url.endswith('/') + if not '/' in service_url: # plain host:port + service_url = 'http://' + service_url + + self.service_url = service_url + + self.username = username + self.password = password + + self.tid = None + + if prev_session: + self.service_url = prev.service_url + + if cookiefile is not None: + cookiejar = self.open_cookie_file(cookiefile) + + if cookiejar is not None: + self.cookiejar = cookiejar + elif prev_session: + self.cookiejar = prev_session.cookiejar + else: + self.cookiejar = self._default_cookiejar + + self._http_request = http_client(self.cookiejar, self._raise_service_error) + + + def open_cookie_file(self, cookiefile=None): + if cookiefile is None or cookiefile == '': + if os.environ.has_key('HOME'): + cookiefile = os.path.join(os.environ['HOME'], '.pyfreebase/cookiejar') + else: + raise MetawebError("no cookiefile specified and no $HOME/.pyfreebase directory" % cookiefile) + + cookiejar = cookielib.LWPCookieJar(cookiefile) + if os.path.exists(cookiefile): + cookiejar.load(ignore_discard=True) + + return cookiejar + + + def _httpreq(self, service_path, method='GET', body=None, form=None, + headers=None): + """ + make an http request to the service. + + form arguments are encoded in the url, even for POST, if a non-form + content-type is given for the body. + + returns a pair (resp, body) + + resp is the response object and may be different depending + on whether urllib2 or httplib2 is in use? + """ + + if method == 'GET': + assert body is None + if method != "GET" and method != "POST": + assert 0, 'unknown method %s' % method + + url = self.service_url + service_path + + if headers is None: + headers = {} + else: + headers = _normalize_headers(headers) + + # this is a lousy way to parse Content-Type, where is the library? + ct = headers.get('content-type', None) + if ct is not None: + ct = ct.split(';')[0] + + if body is not None: + # if body is provided, content-type had better be too + assert ct is not None + + if form is not None: + qstr = '&'.join(['%s=%s' % (urlencode_weak(unicode(k)), urlencode_weak(unicode(v))) + for k,v in form.items()]) + if method == 'POST': + # put the args on the url if we're putting something else + # in the body. this is used to add args to raw uploads. + if body is not None: + url += '?' + qstr + else: + if ct is None: + ct = 'application/x-www-form-urlencoded' + headers['content-type'] = ct + '; charset=utf-8' + + if ct == 'multipart/form-encoded': + # TODO handle this case + raise NotImplementedError + elif ct == 'application/x-www-form-urlencoded': + body = qstr + else: + # for all methods other than POST, use the url + url += '?' + qstr + + + # assure the service that this isn't a CSRF form submission + headers['x-metaweb-request'] = 'Python' + + if 'user-agent' not in headers: + headers['user-agent'] = 'python freebase.api-%s' % __version__ + + #if self.tid is not None: + # headers['x-metaweb-tid'] = self.tid + + ####### DEBUG MESSAGE - should check log level before generating + if form is None: + formstr = '' + else: + formstr = '\nFORM:\n ' + '\n '.join(['%s=%s' % (k,v) + for k,v in form.items()]) + if headers is None: + headerstr = '' + else: + headerstr = '\nHEADERS:\n ' + '\n '.join([('%s: %s' % (k,v)) + for k,v in headers.items()]) + self.log.info('%s %s%s%s', method, url, formstr, headerstr) + ####### + + return self._http_request(url, method, body, headers) + + def _raise_service_error(self, url, status, ctype, body): + + is_jsbody = (ctype.endswith('javascript') + or ctype.endswith('json')) + if str(status) == '400' and is_jsbody: + r = self._loadjson(body) + msg = r.messages[0] + raise MetawebError(u'%s %s %r' % (msg.get('code',''), msg.message, msg.info)) + + raise MetawebError, 'request failed: %s: %r %r' % (url, status, body) + + def _httpreq_json(self, *args, **kws): + resp, body = self._httpreq(*args, **kws) + return self._loadjson(body) + + def _loadjson(self, json): + # TODO really this should be accomplished by hooking + # simplejson to create attrdicts instead of dicts. + def struct2attrdict(st): + """ + copy a json structure, turning all dicts into attrdicts. + + copying descends instances of dict and list, including subclasses. + """ + if isinstance(st, dict): + return attrdict([(k,struct2attrdict(v)) for k,v in st.items()]) + if isinstance(st, list): + return [struct2attrdict(li) for li in st] + return st + + if json == '': + self.log.error('the empty string is not valid json') + raise MetawebError('the empty string is not valid json') + + try: + r = simplejson.loads(json) + except ValueError, e: + self.log.error('error parsing json string %r' % json) + raise MetawebError, 'error parsing JSON string: %s' % e + + return struct2attrdict(r) + + def _check_mqlerror(self, r): + if r.code != '/api/status/ok': + for msg in r.messages: + self.log.error('mql error: %s %s %r' % (msg.code, msg.message, msg.get('query', None))) + raise MetawebError, 'query failed: %s\n%r' % (r.messages[0].code, r.messages[0].get('query', None)) + + def _mqlresult(self, r): + self._check_mqlerror(r) + + self.log.info('result: %s', Delayed(logformat, r)) + + return r.result + + + + def login(self, username=None, password=None): + """sign in to the service. For a more complete description, + see http://www.freebase.com/view/en/api_account_login""" + + service = '/api/account/login' + + username = username or self.username + password = password or self.password + + assert username is not None + assert password is not None + + self.log.debug('LOGIN USERNAME: %s', username) + + r = self._httpreq_json(service, 'POST', + form=dict(username=username, + password=password)) + + if r.code != '/api/status/ok': + raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) + + self.log.debug('LOGIN RESP: %r', r) + self.log.debug('LOGIN COOKIES: %s', self.cookiejar) + + def logout(self): + """logout of the service. For a more complete description, + see http://www.freebase.com/view/en/api_account_logout""" + + service = '/api/account/logout' + + self.log.debug("LOGOUT") + + r = self._httpreq_json(service, 'GET') + + if r.code != '/api/status/ok': + raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) #this should never happen + + def user_info(self, mql_output=None): + """ get user_info. For a more complete description, + see http://www.freebase.com/view/en/api_service_user_info""" + + service = "/api/service/user_info" + + qstr = simplejson.dumps(mql_output) + + r = self._httpreq_json(service, 'POST', form=dict(mql_output=qstr)) + return r + + def loggedin(self): + """check to see whether a user is logged in or not. For a + more complete description, see http://www.freebase.com/view/en/api_account_loggedin""" + + service = "/api/account/loggedin" + try: + r = self._httpreq_json(service, 'GET') + if r.code == "/api/status/ok": + return True + + except MetawebError, me: + return False + + def mqlreaditer(self, sq, asof=None): + """read a structure query.""" + + cursor = True + + while 1: + subq = dict(query=[sq], cursor=cursor, escape=False) + if asof: + subq['as_of_time'] = asof + + qstr = simplejson.dumps(subq) + + service = '/api/service/mqlread' + + r = self._httpreq_json(service, form=dict(query=qstr)) + + for item in self._mqlresult(r): + yield item + + if r['cursor']: + cursor = r['cursor'] + self.log.info('CONTINUING with %s', cursor) + else: + return + + def mqlread(self, sq, asof=None): + """read a structure query. For a more complete description, + see http://www.freebase.com/view/en/api_service_mqlread""" + subq = dict(query=sq, escape=False) + if asof: + subq['as_of_time'] = asof + + if isinstance(sq, list): + subq['cursor'] = True + + service = '/api/service/mqlread' + + self.log.info('%s: %s', + service, + Delayed(logformat, sq)) + + qstr = simplejson.dumps(subq) + r = self._httpreq_json(service, form=dict(query=qstr)) + + return self._mqlresult(r) + + def mqlreadmulti(self, queries, asof=None): + """read a structure query""" + keys = [('q%d' % i) for i,v in enumerate(queries)]; + envelope = {} + for i,sq in enumerate(queries): + subq = dict(query=sq, escape=False) + if asof: + subq['as_of_time'] = asof + + # XXX put this back once mqlreadmulti is working in general + #if isinstance(sq, list): + # subq['cursor'] = True + envelope[keys[i]] = subq + + service = '/api/service/mqlread' + + self.log.info('%s: %s', + service, + Delayed(logformat, envelope)) + + qstr = simplejson.dumps(envelope) + rs = self._httpreq_json(service, form=dict(queries=qstr)) + + self.log.info('%s result: %s', + service, + Delayed(simplejson.dumps, rs, indent=2)) + + return [self._mqlresult(rs[key]) for key in keys] + + def raw(self, id): + """translate blob from id. For a more complete description, + see http://www.freebase.com/view/en/api_trans_raw""" + url = '/api/trans/raw' + urlquote(id) + + self.log.info(url) + + resp, body = self._httpreq(url) + + self.log.info('raw is %d bytes' % len(body)) + + return body + + def blurb(self, id, break_paragraphs=False, maxlength=200): + """translate only the text in blob from id. For a more + complete description, see http://www.freebase.com/view/en/api_trans_blurb""" + url = '/api/trans/blurb' + urlquote(id) + + self.log.info(url) + + resp, body = self._httpreq(url, form=dict(break_paragraphs=break_paragraphs, maxlength=maxlength)) + + self.log.info('blurb is %d bytes' % len(body)) + + return body + + def image_thumb(self, id, maxwidth=None, maxheight=None, mode="fit", onfail=None): + """ given the id of an image, this will return a URL of a thumbnail of the image. + The full details of how the image is cropped and finessed is detailed at + http://www.freebase.com/view/en/api_trans_image_thumb """ + + service = "/api/trans/image_thumb" + assert mode in ["fit", "fill", "fillcrop", "fillcropmid"] + + form = dict(mode=mode) + if maxwidth is not None: + form["maxwidth"] = maxwidth + if maxheight is not None: + form["maxheight"] = maxheight + if onfail is not None: + form["onfail"] = onfail + + resp, body = self._httpreq(service + urlquote(id), form=form) + self.log.info('image is %d bytes' % len(body)) + + return body + + def mqlwrite(self, sq): + """do a mql write. For a more complete description, + see http://www.freebase.com/view/en/api_service_mqlwrite""" + query = dict(query=sq, escape=False) + qstr = simplejson.dumps(query) + + self.log.debug('MQLWRITE: %s', qstr) + + service = '/api/service/mqlwrite' + + self.log.info('%s: %s', + service, + Delayed(logformat,sq)) + + r = self._httpreq_json(service, 'POST', + form=dict(query=qstr)) + + self.log.debug('MQLWRITE RESP: %r', r) + return self._mqlresult(r) + + def mqlcheck(self, sq): + """ See if a write is valid, and see what would happen, but do not + actually do the write """ + + query = dict(query=sq, escape=False) + qstr = simplejson.dumps(query) + + self.log.debug('MQLCHECK: %s', qstr) + + service = '/api/service/mqlcheck' + + self.log.info('%s: %s', + service, + Delayed(logformat, sq)) + + r = self._httpreq_json(service, 'POST', + form=dict(query=qstr)) + + + self.log.debug('MQLCHECK RESP: %r', r) + + return self._mqlresult(r) + + def mqlflush(self): + """ask the service not to hand us old data""" + self.log.debug('MQLFLUSH') + + service = '/api/service/touch' + r = self._httpreq_json(service) + + self._check_mqlerror(r) + return True + + def touch(self): + """ make sure you are accessing the most recent data. For a more + complete description, see http://www.freebase.com/view/en/api_service_touch""" + return self.mqlflush() + + + def upload(self, body, content_type, document_id=False, permission_of=False): + """upload to the metaweb. For a more complete description, + see http://www.freebase.com/view/en/api_service_upload""" + + service = '/api/service/upload' + + self.log.info('POST %s: %s (%d bytes)', + service, content_type, len(body)) + + + headers = {} + if content_type is not None: + headers['content-type'] = content_type + + form = None + if document_id is not False: + if document_id is None: + form = { 'document': '' } + else: + form = { 'document': document_id } + if permission_of is not False: + if form: + form['permission_of'] = permission_of + else: + form = { 'permission_of' : permission_of } + + # note the use of both body and form. + # form parameters get encoded into the URL in this case + r = self._httpreq_json(service, 'POST', + headers=headers, body=body, form=form) + return self._mqlresult(r) + + + def version(self): + """ get versions for various parts of freebase. For a more + complete description, see http://www.freebase.com/view/en/api_version""" + + service = "/api/version" + r = self._httpreq_json(service) + + self._check_mqlerror(r) + return r + + ### DEPRECATED IN API + def reconcile(self, name, etype=['/common/topic']): + """DEPRECATED: reconcile name to guid. For a more complete description, + see http://www.freebase.com/view/en/dataserver_reconciliation""" + + service = '/dataserver/reconciliation' + r = self._httpreq_json(service, 'GET', form={'name':name, 'types':','.join(etype)}) + + + # TODO non-conforming service, fix later + #self._mqlresult(r) + return r + + +if __name__ == '__main__': + console = logging.StreamHandler() + console.setLevel(logging.DEBUG) + + mss = HTTPMetawebSession('sandbox.freebase.com') + + self.mss.log.setLevel(logging.DEBUG) + self.mss.log.addHandler(console) + + + print mss.mqlread([dict(name=None, type='/type/type')]) -- cgit v1.3.1