diff options
| -rw-r--r-- | MANIFEST.in | 2 | ||||
| -rw-r--r-- | examples/get_albums.py | 15 | ||||
| -rw-r--r-- | freebase/api/session.py | 353 | ||||
| -rwxr-xr-x | test/test_all.py | 84 |
4 files changed, 299 insertions, 155 deletions
diff --git a/MANIFEST.in b/MANIFEST.in index b63f69d..3f1ee01 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ recursive-include freebase *.py *.txt +recursive-include examples *.py *.txt +include freebase-README exclude metaweb.py diff --git a/examples/get_albums.py b/examples/get_albums.py index 9d2e46f..2a20cd7 100644 --- a/examples/get_albums.py +++ b/examples/get_albums.py @@ -27,7 +27,12 @@ # POSSIBILITY OF SUCH DAMAGE. # ======================================================================== -from freebase.api import HTTPMetawebSession, MetawebError +import freebase + +print freebase.search("Kurt Vonnegut") + + +#from freebase.api import HTTPMetawebSession, MetawebError #username='username' #password='password' @@ -38,11 +43,11 @@ query = { 'type':'/music/artist', } #credentials = freebase.login(username, password) -mss = HTTPMetawebSession('www.freebase.com') -result = mss.mqlread(query) +#mss = HTTPMetawebSession('www.freebase.com') +#result = mss.mqlread(query) -for album in result.album: - print album +#for album in result.album: +# print album # output : diff --git a/freebase/api/session.py b/freebase/api/session.py index 0d5159b..28fe569 100644 --- a/freebase/api/session.py +++ b/freebase/api/session.py @@ -11,7 +11,7 @@ # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS @@ -29,9 +29,9 @@ """ declarations for external metaweb api. - + from metaweb.api import HTTPMetawebSession - + mss = HTTPMetawebSession('sandbox.freebase.com') print mss.mqlread([dict(name=None, type='/type/type')]) """ @@ -63,13 +63,13 @@ class Delayed(object): """ Wrapper for callables in log statements. Avoids actually making the call until the result is turned into a string. - + A few examples: - + simplejson.dumps is never called because the logger never tries to format the result >>> logging.debug(Delayed(simplejson.dumps, q)) - + This time simplejson.dumps() is actually called: >>> logging.warn(Delayed(simplejson.dumps, q)) @@ -78,10 +78,10 @@ class Delayed(object): self.f = f self.args = args self.kwds = kwds - + def __str__(self): return str(self.f(*self.args, **self.kwds)) - + def logformat(result): """ Format the dict/list as a json object @@ -123,7 +123,7 @@ def urlencode_weak(s): # from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/361668 class attrdict(dict): """A dict whose items can also be accessed as member variables. - + >>> d = attrdict(a=1, b=2) >>> d['c'] = 3 >>> print d.a, d.b, d.c @@ -131,7 +131,7 @@ class attrdict(dict): >>> d.b = 10 >>> print d['b'] 10 - + # but be careful, it's easy to hide methods >>> print d.get('c') 3 @@ -162,12 +162,12 @@ class MetawebSession(object): """ MetawebSession is the base class for MetawebSession, subclassed for different connection types. Only http is available externally. - + This is more of an interface than a class """ - - # interface definition here... + # interface definition here... + # from httplib2 NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') @@ -177,99 +177,99 @@ def _normalize_headers(headers): class HTTPMetawebSession(MetawebSession): """ a MetawebSession is a request/response queue. - + this version uses the HTTP api, and is synchronous. """ # share cookies across sessions, so that different sessions can # see each other's writes immediately. _default_cookiejar = cookielib.CookieJar() - + def __init__(self, service_url, username=None, password=None, prev_session=None, cookiejar=None, cookiefile=None): """ create a new MetawebSession for interacting with the Metaweb. - - a new session will inherit state from prev_session if present, + + a new session will inherit state from prev_session if present, """ super(HTTPMetawebSession, self).__init__() - + self.log = logging.getLogger() - + assert not service_url.endswith('/') if not '/' in service_url: # plain host:port service_url = 'http://' + service_url - + self.service_url = service_url - + self.username = username self.password = password - + self.tid = None - + if prev_session: self.service_url = prev.service_url - + if cookiefile is not None: cookiejar = self.open_cookie_file(cookiefile) - + if cookiejar is not None: self.cookiejar = cookiejar elif prev_session: self.cookiejar = prev_session.cookiejar else: self.cookiejar = self._default_cookiejar - + self._http_request = http_client(self.cookiejar, self._raise_service_error) - + def open_cookie_file(self, cookiefile=None): if cookiefile is None or cookiefile == '': if os.environ.has_key('HOME'): cookiefile = os.path.join(os.environ['HOME'], '.pyfreebase/cookiejar') else: raise MetawebError("no cookiefile specified and no $HOME/.pyfreebase directory" % cookiefile) - + cookiejar = cookielib.LWPCookieJar(cookiefile) if os.path.exists(cookiefile): cookiejar.load(ignore_discard=True) - + return cookiejar - + def _httpreq(self, service_path, method='GET', body=None, form=None, headers=None): """ make an http request to the service. - + form arguments are encoded in the url, even for POST, if a non-form content-type is given for the body. - + returns a pair (resp, body) - + resp is the response object and may be different depending on whether urllib2 or httplib2 is in use? """ - + if method == 'GET': assert body is None if method != "GET" and method != "POST": assert 0, 'unknown method %s' % method - + url = self.service_url + service_path - + if headers is None: headers = {} else: headers = _normalize_headers(headers) - + # this is a lousy way to parse Content-Type, where is the library? ct = headers.get('content-type', None) if ct is not None: ct = ct.split(';')[0] - + if body is not None: # if body is provided, content-type had better be too assert ct is not None - + if form is not None: qstr = '&'.join(['%s=%s' % (urlencode_weak(unicode(k)), urlencode_weak(unicode(v))) for k,v in form.items()]) @@ -282,7 +282,7 @@ class HTTPMetawebSession(MetawebSession): if ct is None: ct = 'application/x-www-form-urlencoded' headers['content-type'] = ct + '; charset=utf-8' - + if ct == 'multipart/form-encoded': # TODO handle this case raise NotImplementedError @@ -292,17 +292,17 @@ class HTTPMetawebSession(MetawebSession): # for all methods other than POST, use the url url += '?' + qstr - + # assure the service that this isn't a CSRF form submission headers['x-metaweb-request'] = 'Python' - + if 'user-agent' not in headers: headers['user-agent'] = 'python freebase.api-%s' % __version__ - + #if self.tid is not None: # headers['x-metaweb-tid'] = self.tid - - ####### DEBUG MESSAGE - should check log level before generating + + ####### DEBUG MESSAGE - should check log level before generating if form is None: formstr = '' else: @@ -315,24 +315,24 @@ class HTTPMetawebSession(MetawebSession): for k,v in headers.items()]) self.log.info('%s %s%s%s', method, url, formstr, headerstr) ####### - + return self._http_request(url, method, body, headers) - + def _raise_service_error(self, url, status, ctype, body): - + is_jsbody = (ctype.endswith('javascript') or ctype.endswith('json')) if str(status) == '400' and is_jsbody: r = self._loadjson(body) msg = r.messages[0] raise MetawebError(u'%s %s %r' % (msg.get('code',''), msg.message, msg.info)) - - raise MetawebError, 'request failed: %s: %r %r' % (url, status, body) + raise MetawebError, 'request failed: %s: %r %r' % (url, status, body) + def _httpreq_json(self, *args, **kws): resp, body = self._httpreq(*args, **kws) return self._loadjson(body) - + def _loadjson(self, json): # TODO really this should be accomplished by hooking # simplejson to create attrdicts instead of dicts. @@ -347,43 +347,43 @@ class HTTPMetawebSession(MetawebSession): if isinstance(st, list): return [struct2attrdict(li) for li in st] return st - + if json == '': self.log.error('the empty string is not valid json') raise MetawebError('the empty string is not valid json') - + try: r = simplejson.loads(json) except ValueError, e: self.log.error('error parsing json string %r' % json) raise MetawebError, 'error parsing JSON string: %s' % e - + return struct2attrdict(r) - + def _check_mqlerror(self, r): if r.code != '/api/status/ok': for msg in r.messages: self.log.error('mql error: %s %s %r' % (msg.code, msg.message, msg.get('query', None))) raise MetawebError, 'query failed: %s\n%r' % (r.messages[0].code, r.messages[0].get('query', None)) - + def _mqlresult(self, r): self._check_mqlerror(r) - + self.log.info('result: %s', Delayed(logformat, r)) - + return r.result - + def login(self, username=None, password=None): """sign in to the service. For a more complete description, see http://www.freebase.com/view/en/api_account_login""" - + service = '/api/account/login' - + username = username or self.username password = password or self.password - + assert username is not None assert password is not None @@ -392,13 +392,13 @@ class HTTPMetawebSession(MetawebSession): r = self._httpreq_json(service, 'POST', form=dict(username=username, password=password)) - + if r.code != '/api/status/ok': raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) - + self.log.debug('LOGIN RESP: %r', r) self.log.debug('LOGIN COOKIES: %s', self.cookiejar) - + def logout(self): """logout of the service. For a more complete description, see http://www.freebase.com/view/en/api_account_logout""" @@ -411,7 +411,7 @@ class HTTPMetawebSession(MetawebSession): if r.code != '/api/status/ok': raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) #this should never happen - + def user_info(self, mql_output=None): """ get user_info. For a more complete description, see http://www.freebase.com/view/en/api_service_user_info""" @@ -432,56 +432,56 @@ class HTTPMetawebSession(MetawebSession): r = self._httpreq_json(service, 'GET') if r.code == "/api/status/ok": return True - + except MetawebError, me: return False - + def mqlreaditer(self, sq, asof=None): """read a structure query.""" - + cursor = True - + while 1: subq = dict(query=[sq], cursor=cursor, escape=False) if asof: subq['as_of_time'] = asof - + qstr = simplejson.dumps(subq) - + service = '/api/service/mqlread' r = self._httpreq_json(service, form=dict(query=qstr)) - + for item in self._mqlresult(r): yield item - + if r['cursor']: cursor = r['cursor'] self.log.info('CONTINUING with %s', cursor) else: return - + def mqlread(self, sq, asof=None): """read a structure query. For a more complete description, see http://www.freebase.com/view/en/api_service_mqlread""" subq = dict(query=sq, escape=False) if asof: subq['as_of_time'] = asof - + if isinstance(sq, list): subq['cursor'] = True - + service = '/api/service/mqlread' - + self.log.info('%s: %s', service, Delayed(logformat, sq)) - + qstr = simplejson.dumps(subq) r = self._httpreq_json(service, form=dict(query=qstr)) - + return self._mqlresult(r) - + def mqlreadmulti(self, queries, asof=None): """read a structure query""" keys = [('q%d' % i) for i,v in enumerate(queries)]; @@ -490,58 +490,58 @@ class HTTPMetawebSession(MetawebSession): subq = dict(query=sq, escape=False) if asof: subq['as_of_time'] = asof - + # XXX put this back once mqlreadmulti is working in general #if isinstance(sq, list): # subq['cursor'] = True envelope[keys[i]] = subq - + service = '/api/service/mqlread' - + self.log.info('%s: %s', service, Delayed(logformat, envelope)) - + qstr = simplejson.dumps(envelope) rs = self._httpreq_json(service, form=dict(queries=qstr)) - + self.log.info('%s result: %s', service, Delayed(simplejson.dumps, rs, indent=2)) - + return [self._mqlresult(rs[key]) for key in keys] - + def raw(self, id): """translate blob from id. For a more complete description, see http://www.freebase.com/view/en/api_trans_raw""" url = '/api/trans/raw' + urlquote(id) - + self.log.info(url) - + resp, body = self._httpreq(url) - + self.log.info('raw is %d bytes' % len(body)) - + return body def blurb(self, id, break_paragraphs=False, maxlength=200): - """translate only the text in blob from id. For a more + """translate only the text in blob from id. For a more complete description, see http://www.freebase.com/view/en/api_trans_blurb""" url = '/api/trans/blurb' + urlquote(id) - + self.log.info(url) - + resp, body = self._httpreq(url, form=dict(break_paragraphs=break_paragraphs, maxlength=maxlength)) - + self.log.info('blurb is %d bytes' % len(body)) - + return body - + def image_thumb(self, id, maxwidth=None, maxheight=None, mode="fit", onfail=None): """ given the id of an image, this will return a URL of a thumbnail of the image. - The full details of how the image is cropped and finessed is detailed at + The full details of how the image is cropped and finessed is detailed at http://www.freebase.com/view/en/api_trans_image_thumb """ - + service = "/api/trans/image_thumb" assert mode in ["fit", "fill", "fillcrop", "fillcropmid"] @@ -557,80 +557,80 @@ class HTTPMetawebSession(MetawebSession): self.log.info('image is %d bytes' % len(body)) return body - + def mqlwrite(self, sq): """do a mql write. For a more complete description, see http://www.freebase.com/view/en/api_service_mqlwrite""" query = dict(query=sq, escape=False) qstr = simplejson.dumps(query) - + self.log.debug('MQLWRITE: %s', qstr) - + service = '/api/service/mqlwrite' - + self.log.info('%s: %s', service, Delayed(logformat,sq)) - + r = self._httpreq_json(service, 'POST', form=dict(query=qstr)) - + self.log.debug('MQLWRITE RESP: %r', r) return self._mqlresult(r) - + def mqlcheck(self, sq): """ See if a write is valid, and see what would happen, but do not actually do the write """ query = dict(query=sq, escape=False) qstr = simplejson.dumps(query) - + self.log.debug('MQLCHECK: %s', qstr) - + service = '/api/service/mqlcheck' - + self.log.info('%s: %s', service, Delayed(logformat, sq)) - + r = self._httpreq_json(service, 'POST', form=dict(query=qstr)) - + self.log.debug('MQLCHECK RESP: %r', r) - + return self._mqlresult(r) def mqlflush(self): """ask the service not to hand us old data""" self.log.debug('MQLFLUSH') - + service = '/api/service/touch' r = self._httpreq_json(service) - + self._check_mqlerror(r) return True - + def touch(self): """ make sure you are accessing the most recent data. For a more complete description, see http://www.freebase.com/view/en/api_service_touch""" return self.mqlflush() - + def upload(self, body, content_type, document_id=False, permission_of=False): """upload to the metaweb. For a more complete description, see http://www.freebase.com/view/en/api_service_upload""" - + service = '/api/service/upload' - + self.log.info('POST %s: %s (%d bytes)', service, content_type, len(body)) - + headers = {} if content_type is not None: headers['content-type'] = content_type - + form = None if document_id is not False: if document_id is None: @@ -642,13 +642,105 @@ class HTTPMetawebSession(MetawebSession): form['permission_of'] = permission_of else: form = { 'permission_of' : permission_of } - + # note the use of both body and form. # form parameters get encoded into the URL in this case r = self._httpreq_json(service, 'POST', headers=headers, body=body, form=form) return self._mqlresult(r) - + + + def search(self, query, format=None, prefixed=None, limit=20, start=0, + type=None, type_strict="any", domain=None, domain_strict=None, + escape="html", timeout=None, mql_filter=None, mql_output=None): + """ search freebase.com. For a more complete description, + see http://www.freebase.com/view/en/api_service_search""" + + service = "/api/service/search" + + form = dict(query=query) + + if format: + form["format"] = format + if prefixed: + form["prefixed"] = prefixed + if limit: + form["limit"] = limit + if start: + form["start"] = start + if type: + form["type"] = type + if type_strict: + form["type_strict"] = type_strict + if domain: + form["domain"] = domain + if domain_strict: + form["domain_strict"] = domain_strict + if escape: + form["escape"] = escape + if timeout: + form["timeout"] = timeout + if mql_filter: + form["mql_filter"] = mql_filter + if mql_output: + form["mql_output"] = mql_output + + + r = self._httpreq_json(service, 'POST', form=form) + + return self._mqlresult(r) + + + def geosearch(self, location=None, location_type=None, mql_input=None, limit=20, + start=0, type=None, geometry_type=None, intersect=None, mql_filter=None, + within=None, inside=None, order_by=None, count=None, format="json", mql_output=None): + """ perform a geosearch. For a more complete description, + see http://www.freebase.com/api/service/geosearch?help """ + + service = "/api/service/geosearch" + + if location is None and location_type is None and mql_input is None: + raise Exception("You have to give it something to work with") + + form = dict() + + if location: + form["location"] = location + if location_type: + form["location_type"] = location_type + if mql_input: + form["mql_input"] = mql_input + if limit: + form["limit"] = limit + if start: + form["start"] = start + if type: + form["type"] = type + if geometry_type: + form["geometry_type"] = geometry_type + if intersect: + form["intersect"] = intersect + if mql_filter: + form["mql_filter"] = mql_filter + if within: + form["within"] = within + if inside: + form["inside"] = inside + if order_by: + form["order_by"] = order_by + if count: + form["count"] = count + if format: + form["format"] = format + if mql_output: + form["mql_output"] = mql_output + + if format == "json": + r = self._httpreq_json(service, 'POST', form=form) + else: + r = self._httpreq(service, 'POST', form=form) + + return r def version(self): """ get versions for various parts of freebase. For a more @@ -657,31 +749,30 @@ class HTTPMetawebSession(MetawebSession): service = "/api/version" r = self._httpreq_json(service) - self._check_mqlerror(r) - return r + return self._mqlresult(r) ### DEPRECATED IN API def reconcile(self, name, etype=['/common/topic']): """DEPRECATED: reconcile name to guid. For a more complete description, see http://www.freebase.com/view/en/dataserver_reconciliation""" - + service = '/dataserver/reconciliation' r = self._httpreq_json(service, 'GET', form={'name':name, 'types':','.join(etype)}) - + # TODO non-conforming service, fix later #self._mqlresult(r) return r - + if __name__ == '__main__': console = logging.StreamHandler() console.setLevel(logging.DEBUG) - + mss = HTTPMetawebSession('sandbox.freebase.com') - + self.mss.log.setLevel(logging.DEBUG) self.mss.log.addHandler(console) - + print mss.mqlread([dict(name=None, type='/type/type')]) diff --git a/test/test_all.py b/test/test_all.py index 1a9e124..cf52eef 100755 --- a/test/test_all.py +++ b/test/test_all.py @@ -12,7 +12,7 @@ # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @@ -59,12 +59,12 @@ class TestFreebase(unittest.TestCase): self.assertNotEqual(None, user_info) self.assertEqual(user_info.code, "/api/status/ok") self.assertEqual(mss.loggedin(), True) - + mss.logout() self.assertRaises(MetawebError, mss.user_info) self.assertEqual(mss.loggedin(), False) - + def test_freebase_dot_read(self): query = {'type':'/music/artist','guid':[{}],'name':'Sting', 'album':[{}]} @@ -79,16 +79,16 @@ class TestFreebase(unittest.TestCase): self.assert_(len(result['album']) > 0) self.assertEqual( 'Sting', result['name']) self.assertEqual('#9202a8c04000641f8000000000092a01', result['guid'][0]['value']) - + def test_freebase_dot_write(self): read_query = {'type':'/music/artist','name':'Yanni\'s Cousin Tom', 'id':{}} freebase.sandbox.login(username=USERNAME, password=PASSWORD) result = freebase.sandbox.mqlread(read_query) self.assertEqual(None, result) - + write_query = {'create':'unless_exists', 'type':'/music/artist','name':'Yanni'} - + write_result = freebase.sandbox.mqlwrite(write_query) self.assertNotEqual(None, write_result) self.assert_(write_result.has_key('create')) @@ -96,13 +96,13 @@ class TestFreebase(unittest.TestCase): self.assert_(write_result.has_key('name')) self.assertEqual('existed', write_result['create']) self.assertEqual('Yanni', write_result['name']) - self.assertEqual('/music/artist', write_result['type']) - + self.assertEqual('/music/artist', write_result['type']) + def test_read(self): query = {'type':'/music/artist','guid':[{}],'name':'Sting', 'album':[{}]} - + mss = HTTPMetawebSession(API_HOST) - + result = mss.mqlread(query) self.assertNotEqual(None, result) @@ -114,17 +114,17 @@ class TestFreebase(unittest.TestCase): self.assert_(len(result['album']) > 0) self.assertEqual( 'Sting', result['name']) self.assertEqual('#9202a8c04000641f8000000000092a01', result['guid'][0]['value']) - + def test_write(self): read_query = {'type':'/music/artist','name':'Yanni\'s Cousin Tom', 'id':{}} mss = HTTPMetawebSession(API_HOST, username=USERNAME, password=PASSWORD) result = mss.mqlread(read_query) self.assertEqual(None, result) - + write_query = {'create':'unless_exists', 'type':'/music/artist','name':'Yanni'} - + mss.login() write_result = mss.mqlwrite(write_query) self.assertNotEqual(None, write_result) @@ -134,7 +134,7 @@ class TestFreebase(unittest.TestCase): self.assertEqual('existed', write_result['create']) self.assertEqual('Yanni', write_result['name']) self.assertEqual('/music/artist', write_result['type']) - + def test_trans_blurb(self): kurt = "/en/kurt_vonnegut" @@ -153,12 +153,12 @@ class TestFreebase(unittest.TestCase): def test_trans_raw(self): kurt = "/en/kurt_vonnegut" - + self.assertRaises(MetawebError, lambda: freebase.raw(kurt)) r = freebase.mqlread({"id":kurt, "/common/topic/article":[{"id":None, "optional":True, "limit":1}]}) raw = freebase.raw(r["/common/topic/article"][0].id) - self.assertNotEqual(len(raw), 0) + self.assertNotEqual(len(raw), 0) def test_trans_image_thumb(self): kurt = "/en/kurt_vonnegut" @@ -179,12 +179,58 @@ class TestFreebase(unittest.TestCase): self.assertEqual(freebase.sandbox.raw(response.id), my_text) # since it's text/plain, blurb should also be equal self.assertEqual(freebase.sandbox.blurb(response.id), my_text) + + + def is_kurt_there(self, results): + for result in results: + if result.name == "Kurt Vonnegut": + return True + return False + + + def test_search(self): + r0 = freebase.search("Kurt V") + self.assertEqual(self.is_kurt_there(r0), True) + + r1 = freebase.search("Kurt V", type=["/location/citytown"]) + self.assertEqual(self.is_kurt_there(r1), False) + + r2 = freebase.search("Kurt V", type=["/location/citytown", "/music/artist"]) + self.assertEqual(self.is_kurt_there(r2), False) + + self.assertNotEqual(len(r0), len(r1)) + self.assertNotEqual(len(r0), len(r2)) + self.assertNotEqual(len(r1), len(r2)) + + def test_touch(self): + pass + + + def test_geosearch(self): + self.assertRaises(Exception, freebase.geosearch) + r0 = freebase.geosearch(location="/en/california") + self.assertNotEqual(len(r0), 0) + + + + + if __name__ == '__main__': if USERNAME == "username" and PASSWORD == "password": - print "In order to run the tests, we need to use a valid freebase username and password" - USERNAME = raw_input("Please enter your username: ") - PASSWORD = raw_input("Please enter your password (it'll appear in cleartext): ") + + try: + passwordfile = open("test/.password.txt", "r") + fh = passwordfile.read().split("\n") + USERNAME = fh[0] + PASSWORD = fh[1] + passwordfile.close() + + except Exception, e: + print "In order to run the tests, we need to use a valid freebase username and password" + USERNAME = raw_input("Please enter your username: ") + PASSWORD = raw_input("Please enter your password (it'll appear in cleartext): ") + unittest.main() |