diff options
| author | nitromaster101 <nitromaster101@5914aa95-5b3a-0410-a3b5-7b719e7fe9b2> | 2009-06-17 19:02:50 +0000 |
|---|---|---|
| committer | nitromaster101 <nitromaster101@5914aa95-5b3a-0410-a3b5-7b719e7fe9b2> | 2009-06-17 19:02:50 +0000 |
| commit | dd2da52625e890e29a6fdbba43456dd8475d51ac (patch) | |
| tree | 1929e92309255c54d0b1bee11489872824f93739 /freebase-api/freebase | |
| parent | cf63c2faee674cc40e1b38975ee05125ee09a232 (diff) | |
reorganize package inorder to emphasize freebase-api as the main one
git-svn-id: http://freebase-python.googlecode.com/svn/trunk@88 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2
Diffstat (limited to 'freebase-api/freebase')
| -rwxr-xr-x | freebase-api/freebase/__init__.py | 40 | ||||
| -rwxr-xr-x | freebase-api/freebase/api/__init__.py | 4 | ||||
| -rwxr-xr-x | freebase-api/freebase/api/cookie_handlers.py | 215 | ||||
| -rw-r--r-- | freebase-api/freebase/api/httpclients.py | 91 | ||||
| -rwxr-xr-x | freebase-api/freebase/api/mqlkey.py | 135 | ||||
| -rw-r--r-- | freebase-api/freebase/api/session.py | 687 | ||||
| -rw-r--r-- | freebase-api/freebase/fcl/README.txt | 8 | ||||
| -rwxr-xr-x | freebase-api/freebase/fcl/__init__.py | 0 | ||||
| -rwxr-xr-x | freebase-api/freebase/fcl/commands.py | 688 | ||||
| -rwxr-xr-x | freebase-api/freebase/fcl/fbutil.py | 113 | ||||
| -rwxr-xr-x | freebase-api/freebase/fcl/fcl.py | 276 | ||||
| -rwxr-xr-x | freebase-api/freebase/fcl/inspect.py | 218 | ||||
| -rwxr-xr-x | freebase-api/freebase/fcl/mktype.py | 179 | ||||
| -rwxr-xr-x | freebase-api/freebase/rison.py | 308 | ||||
| -rw-r--r-- | freebase-api/freebase/sandbox.py | 65 | ||||
| -rwxr-xr-x | freebase-api/freebase/uritemplate.py | 143 |
16 files changed, 0 insertions, 3170 deletions
diff --git a/freebase-api/freebase/__init__.py b/freebase-api/freebase/__init__.py deleted file mode 100755 index 3cca4ff..0000000 --- a/freebase-api/freebase/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -import sys - -from freebase.api.session import HTTPMetawebSession -import sandbox - -__all__ = ["HTTPMetawebSession", "sandbox"] - -base = HTTPMetawebSession("freebase.com") - -# we want to add base's functions to __init__.py -# so that we can say freebase.func() and really -# just call base.func() - -# a little trick to refer to __init__ -# self isn't defined because __init__ is in -# a world in and of itself -self = sys.modules[__name__] - -for funcname in dir(base): - - # we only want the 'real' functions - if not funcname.startswith("_"): - func = getattr(base, funcname) - - # let's make sure we're getting functions - # instead of constants or whatever - if callable(func): - - # we're setting these functions - # so that they can be called like - # freebase.funcname -> base.func() - setattr(self, funcname, func) - - # make sure we import the base's - # functions if we import freebase - __all__.append(funcname) - -# we don't want any self-referencing -# business going. Plus, this is cleaner. -del self diff --git a/freebase-api/freebase/api/__init__.py b/freebase-api/freebase/api/__init__.py deleted file mode 100755 index c3adebc..0000000 --- a/freebase-api/freebase/api/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -from session import HTTPMetawebSession, MetawebError, attrdict - -from mqlkey import quotekey, unquotekey diff --git a/freebase-api/freebase/api/cookie_handlers.py b/freebase-api/freebase/api/cookie_handlers.py deleted file mode 100755 index 88f6b4f..0000000 --- a/freebase-api/freebase/api/cookie_handlers.py +++ /dev/null @@ -1,215 +0,0 @@ -# ======================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ======================================================================== - -# -# -# httplib2cookie.py allows you to use python's standard -# CookieJar class with httplib2. -# -# - -import re - -try: - from google.appengine.api import urlfetch - Http = object -except ImportError: - pass - -try: - from httplib2 import Http -except ImportError: - pass - -try: - import urllib -except ImportError: - import urllib_stub as urllib -import cookielib - -class DummyRequest(object): - """Simulated urllib2.Request object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, url, headers=None): - self.url = url - self.headers = headers - self.origin_req_host = cookielib.request_host(self) - self.type, r = urllib.splittype(url) - self.host, r = urllib.splithost(r) - if self.host: - self.host = urllib.unquote(self.host) - - def get_full_url(self): - return self.url - - def get_origin_req_host(self): - # TODO to match urllib2 this should be different for redirects - return self.origin_req_host - - def get_type(self): - return self.type - - def get_host(self): - return self.host - - def get_header(self, key, default=None): - return self.headers.get(key.lower(), default) - - def has_header(self, key): - return key in self.headers - - def add_unredirected_header(self, key, val): - # TODO this header should not be sent on redirect - self.headers[key.lower()] = val - - def is_unverifiable(self): - # TODO to match urllib2, this should be set to True when the - # request is the result of a redirect - return False - -class DummyHttplib2Response(object): - """Simulated urllib2.Request object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def info(self): - return DummyHttplib2Message(self.response) - - -class DummyUrlfetchResponse(object): - """Simulated urllib2.Request object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def info(self): - return DummyUrlfetchMessage(self.response) - - -class DummyHttplib2Message(object): - """Simulated mimetools.Message object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def getheaders(self, k): - k = k.lower() - v = self.response.get(k.lower(), None) - if k not in self.response: - return [] - #return self.response[k].split(re.compile(',\\s*')) - - # httplib2 joins multiple values for the same header - # using ','. but the netscape cookie format uses ',' - # as part of the expires= date format. so we have - # to split carefully here - header.split(',') won't do it. - HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') - return [h[0] for h in HEADERVAL.findall(self.response[k])] - -class DummyUrlfetchMessage(object): - """Simulated mimetools.Message object for httplib2 - - implements only what's necessary for cookielib.CookieJar to work - """ - def __init__(self, response): - self.response = response - - def getheaders(self, k): - k = k.lower() - v = self.response.headers.get(k.lower(), None) - if k not in self.response.headers: - return [] - #return self.response[k].split(re.compile(',\\s*')) - - # httplib2 joins multiple values for the same header - # using ','. but the netscape cookie format uses ',' - # as part of the expires= date format. so we have - # to split carefully here - header.split(',') won't do it. - HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') - return [h[0] for h in HEADERVAL.findall(self.response.headers[k])] - -class CookiefulHttp(Http): - """Subclass of httplib2.Http that keeps cookie state - - constructor takes an optional cookiejar=cookielib.CookieJar - - currently this does not handle redirects completely correctly: - if the server redirects to a different host the original - cookies will still be sent to that host. - """ - def __init__(self, cookiejar=None, **kws): - # note that httplib2.Http is not a new-style-class - Http.__init__(self, **kws) - if cookiejar is None: - cookiejar = cookielib.CookieJar() - self.cookiejar = cookiejar - - def request(self, uri, **kws): - headers = kws.pop('headers', None) - req = DummyRequest(uri, headers) - self.cookiejar.add_cookie_header(req) - headers = req.headers - - (r, body) = Http.request(self, uri, headers=headers, **kws) - - resp = DummyHttplib2Response(r) - self.cookiejar.extract_cookies(resp, req) - - return (r, body) - -class CookiefulUrlfetch(object): - """Class that keeps cookie state - - constructor takes an optional cookiejar=cookielib.CookieJar - """ - # TODO refactor CookefulHttp so that CookiefulUrlfetch can be a subclass of it - def __init__(self, cookiejar=None, **kws): - if cookiejar is None: - cookejar = cookielib.CookieJar() - self.cookejar = cookiejar - - def request(self, uri, **kws): - headers = kws.pop('headers', None) - req = DummyRequest(uri, headers) - self.cookejar.add_cookie_header(req) - headers = req.headers - - r = urlfetch.fetch(uri, headers=headers, **kws) - - self.cookejar.extract_cookies(DummyUrlfetchResponse(r), req) - return r - diff --git a/freebase-api/freebase/api/httpclients.py b/freebase-api/freebase/api/httpclients.py deleted file mode 100644 index 48af9b6..0000000 --- a/freebase-api/freebase/api/httpclients.py +++ /dev/null @@ -1,91 +0,0 @@ -try: - from google.appengine.api import urlfetch - from cookie_handlers import CookiefulUrlfetch -except: - pass - -try: - import httplib2 - from cookie_handlers import CookiefulHttp -except: - pass - -try: - import urllib2 - import socket -except: - pass - -import logging -import re - -class Urllib2Client(object): - def __init__(self, cookiejar, rse): - cookiespy = urllib2.HTTPCookieProcessor(cookiejar) - self.opener = urllib2.build_opener(cookiespy) - self._raise_service_error = rse - self.log = logging.getLogger() - - def __call__(self, url, method, body, headers): - req = urllib2.Request(url, body, headers) - - try: - resp = self.opener.open(req) - - except socket.error, e: - self.log.error('SOCKET FAILURE: %s', e.fp.read()) - raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) - - except urllib2.HTTPError, e: - self.log.error('HTTP ERROR: %s', e) - self._raise_service_error(url, e.code, e.info().type, e.fp.read()) - - for header in resp.info().headers: - self.log.debug('HTTP HEADER %s', header) - name, value = re.split("[:\n\r]", header, 1) - if name.lower() == 'x-metaweb-tid': - self.tid = value.strip() - - return (resp, resp.read()) - -class Httplib2Client(object): - def __init__(self, cookiejar, rse): - self.cookiejar = cookiejar - self._raise_service_error = rse - self.httpclient = CookiefulHttp(cookiejar=self.cookiejar) - - def __call__(self, url, method, body, headers): - try: - resp, content = self.httpclient.request(url, method=method, - body=body, headers=headers) - if (resp.status != 200): - self._raise_service_error(url, resp.status, resp['content-type'], content) - - except socket.error, e: - self.log.error('SOCKET FAILURE: %s', e.fp.read()) - raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) - - except httplib2.HttpLib2ErrorWithResponse, e: - self._raise_service_error(url, resp.status, resp['content-type'], content) - except httplib2.HttpLib2Error, e: - raise MetawebError(u'HTTP error: %s' % (e,)) - - #tid = resp.get('x-metaweb-tid', None) - - return (resp, content) - - -class UrlfetchClient(object): - def __init__(self, cookiejar, rse): - self.cookiejar = cookiejar - self._raise_service_error = rse - self.httpclient = CookiefulUrlfetch(cookiejar=self.cookiejar) - - def __call__(self, url, method, body, headers): - resp = self.httpclient.request(url, payload=body, method=method, headers=headers) - - if resp.status_code != 200: - self._raise_service_error(url, resp.status_code, resp.headers['content-type'], resp.body) - - return (resp, resp.content) - diff --git a/freebase-api/freebase/api/mqlkey.py b/freebase-api/freebase/api/mqlkey.py deleted file mode 100755 index d7dd01b..0000000 --- a/freebase-api/freebase/api/mqlkey.py +++ /dev/null @@ -1,135 +0,0 @@ -# ======================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ======================================================================== - -import string -import re - -def quotekey(ustr): - """ - quote a unicode string to turn it into a valid namespace key - - """ - valid_always = string.ascii_letters + string.digits - valid_interior_only = valid_always + '_-' - - if isinstance(ustr, str): - s = unicode(ustr,'utf-8') - elif isinstance(ustr, unicode): - s = ustr - else: - raise ValueError, 'quotekey() expects utf-8 string or unicode' - - output = [] - if s[0] in valid_always: - output.append(s[0]) - else: - output.append('$%04X' % ord(s[0])) - - for c in s[1:-1]: - if c in valid_interior_only: - output.append(c) - else: - output.append('$%04X' % ord(c)) - - if len(s) > 1: - if s[-1] in valid_always: - output.append(s[-1]) - else: - output.append('$%04X' % ord(s[-1])) - - return str(''.join(output)) - - -def unquotekey(key, encoding=None): - """ - unquote a namespace key and turn it into a unicode string - """ - - valid_always = string.ascii_letters + string.digits - - output = [] - i = 0 - while i < len(key): - if key[i] in valid_always: - output.append(key[i]) - i += 1 - elif key[i] in '_-' and i != 0 and i != len(key): - output.append(key[i]) - i += 1 - elif key[i] == '$' and i+4 < len(key): - # may raise ValueError if there are invalid characters - output.append(unichr(int(key[i+1:i+5],16))) - i += 5 - else: - raise ValueError, "unquote key saw invalid character '%s' at position %d" % (key[i], i) - - ustr = u''.join(output) - - if encoding is None: - return ustr - - return ustr.encode(encoding) - - -# should this also include "'()" into safe? -def urlencode_pathseg(data): - ''' - urlencode for placement between slashes in an url. - ''' - if isinstance(data, unicode): - data = data.encode('utf_8') - return urllib.quote(data, "~:@$!*,;=&+") - - -def id_to_urlid(id): - """ - convert a mql id to an id suitable for embedding in a url path. - """ - - segs = id.split('/') - - assert isinstance(id, str) and id != '', 'bad id "%s"' % id - - if id[0] == '~': - assert len(segs) == 1 - # assume valid, should check - return id - - if id[0] == '#': - assert len(segs) == 1 - # assume valid, should check - return '%23' + id[1:] - - if id[0] != '/': - raise ValueError, 'unknown id format %s' % id - - # ok, we have a slash-path - # requote components as keys and rejoin. - # urlids do not have leading slashes!!! - return '/'.join(urlencode_pathseg(unquotekey(seg)) for seg in segs[1:]) - diff --git a/freebase-api/freebase/api/session.py b/freebase-api/freebase/api/session.py deleted file mode 100644 index 0d5159b..0000000 --- a/freebase-api/freebase/api/session.py +++ /dev/null @@ -1,687 +0,0 @@ -# ================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ==================================================================== - -""" -declarations for external metaweb api. - - - from metaweb.api import HTTPMetawebSession - - mss = HTTPMetawebSession('sandbox.freebase.com') - print mss.mqlread([dict(name=None, type='/type/type')]) -""" - - - -__all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict'] -__version__ = '0.1' - -import os, sys, re -import cookielib -try: - import simplejson -except ImportError: - try: - # appengine provides simplejson at django.utils.simplejson - from django.utils import simplejson - except ImportError: - raise Exception("unable to import simplejson") -try: - from urllib import quote as urlquote -except ImportError: - from urlib_stub import quote as urlquote -import pprint -import socket -import logging - -class Delayed(object): - """ - Wrapper for callables in log statements. Avoids actually making - the call until the result is turned into a string. - - A few examples: - - simplejson.dumps is never called because the logger never - tries to format the result - >>> logging.debug(Delayed(simplejson.dumps, q)) - - This time simplejson.dumps() is actually called: - >>> logging.warn(Delayed(simplejson.dumps, q)) - - """ - def __init__(self, f, *args, **kwds): - self.f = f - self.args = args - self.kwds = kwds - - def __str__(self): - return str(self.f(*self.args, **self.kwds)) - -def logformat(result): - """ - Format the dict/list as a json object - """ - rstr = simplejson.dumps(result, indent=2) - if rstr[0] == '{': - rstr = rstr[1:-2] - return rstr - -from httpclients import Httplib2Client, Urllib2Client, UrlfetchClient - -# Check for urlfetch first so that urlfetch is used when running the appengine SDK -try: - import google.appengine.api.urlfetch - from cookie_handlers import CookiefulUrlfetch - http_client = UrlfetchClient -except ImportError: - try: - import httplib2 - from cookie_handlers import CookiefulHttp - http_client = Httplib2Client - except ImportError: - import urllib2 - httplib2 = None - CookiefulHttp = None - http_client = Urllib2Client - -# remove whitespace from json encoded output -simplejson.JSONEncoder.item_separator = ',' -simplejson.JSONEncoder.key_separator = ':' -# don't escape slashes, we're not pasting into script tags here. -if simplejson.encoder.ESCAPE_DCT.get('/', None) == r'\/': - simplejson.encoder.ESCAPE_DCT['/'] = '/' - -def urlencode_weak(s): - return urlquote(s, safe=',/:$') - - -# from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/361668 -class attrdict(dict): - """A dict whose items can also be accessed as member variables. - - >>> d = attrdict(a=1, b=2) - >>> d['c'] = 3 - >>> print d.a, d.b, d.c - 1 2 3 - >>> d.b = 10 - >>> print d['b'] - 10 - - # but be careful, it's easy to hide methods - >>> print d.get('c') - 3 - >>> d['get'] = 4 - >>> print d.get('a') - Traceback (most recent call last): - TypeError: 'int' object is not callable - """ - def __init__(self, *args, **kwargs): - dict.__init__(self, *args, **kwargs) - self.__dict__ = self - - - -# TODO expose the common parts of the result envelope -class MetawebError(Exception): - """ - an error report from the metaweb service. - """ - pass - - - -# TODO right now this is a completely unnecessary superclass. -# is there enough common behavior between session types -# to justify it? -class MetawebSession(object): - """ - MetawebSession is the base class for MetawebSession, subclassed for - different connection types. Only http is available externally. - - This is more of an interface than a class - """ - - # interface definition here... - - -# from httplib2 -NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') -def _normalize_headers(headers): - return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) - -class HTTPMetawebSession(MetawebSession): - """ - a MetawebSession is a request/response queue. - - this version uses the HTTP api, and is synchronous. - """ - # share cookies across sessions, so that different sessions can - # see each other's writes immediately. - _default_cookiejar = cookielib.CookieJar() - - def __init__(self, service_url, username=None, password=None, prev_session=None, cookiejar=None, cookiefile=None): - """ - create a new MetawebSession for interacting with the Metaweb. - - a new session will inherit state from prev_session if present, - """ - super(HTTPMetawebSession, self).__init__() - - self.log = logging.getLogger() - - assert not service_url.endswith('/') - if not '/' in service_url: # plain host:port - service_url = 'http://' + service_url - - self.service_url = service_url - - self.username = username - self.password = password - - self.tid = None - - if prev_session: - self.service_url = prev.service_url - - if cookiefile is not None: - cookiejar = self.open_cookie_file(cookiefile) - - if cookiejar is not None: - self.cookiejar = cookiejar - elif prev_session: - self.cookiejar = prev_session.cookiejar - else: - self.cookiejar = self._default_cookiejar - - self._http_request = http_client(self.cookiejar, self._raise_service_error) - - - def open_cookie_file(self, cookiefile=None): - if cookiefile is None or cookiefile == '': - if os.environ.has_key('HOME'): - cookiefile = os.path.join(os.environ['HOME'], '.pyfreebase/cookiejar') - else: - raise MetawebError("no cookiefile specified and no $HOME/.pyfreebase directory" % cookiefile) - - cookiejar = cookielib.LWPCookieJar(cookiefile) - if os.path.exists(cookiefile): - cookiejar.load(ignore_discard=True) - - return cookiejar - - - def _httpreq(self, service_path, method='GET', body=None, form=None, - headers=None): - """ - make an http request to the service. - - form arguments are encoded in the url, even for POST, if a non-form - content-type is given for the body. - - returns a pair (resp, body) - - resp is the response object and may be different depending - on whether urllib2 or httplib2 is in use? - """ - - if method == 'GET': - assert body is None - if method != "GET" and method != "POST": - assert 0, 'unknown method %s' % method - - url = self.service_url + service_path - - if headers is None: - headers = {} - else: - headers = _normalize_headers(headers) - - # this is a lousy way to parse Content-Type, where is the library? - ct = headers.get('content-type', None) - if ct is not None: - ct = ct.split(';')[0] - - if body is not None: - # if body is provided, content-type had better be too - assert ct is not None - - if form is not None: - qstr = '&'.join(['%s=%s' % (urlencode_weak(unicode(k)), urlencode_weak(unicode(v))) - for k,v in form.items()]) - if method == 'POST': - # put the args on the url if we're putting something else - # in the body. this is used to add args to raw uploads. - if body is not None: - url += '?' + qstr - else: - if ct is None: - ct = 'application/x-www-form-urlencoded' - headers['content-type'] = ct + '; charset=utf-8' - - if ct == 'multipart/form-encoded': - # TODO handle this case - raise NotImplementedError - elif ct == 'application/x-www-form-urlencoded': - body = qstr - else: - # for all methods other than POST, use the url - url += '?' + qstr - - - # assure the service that this isn't a CSRF form submission - headers['x-metaweb-request'] = 'Python' - - if 'user-agent' not in headers: - headers['user-agent'] = 'python freebase.api-%s' % __version__ - - #if self.tid is not None: - # headers['x-metaweb-tid'] = self.tid - - ####### DEBUG MESSAGE - should check log level before generating - if form is None: - formstr = '' - else: - formstr = '\nFORM:\n ' + '\n '.join(['%s=%s' % (k,v) - for k,v in form.items()]) - if headers is None: - headerstr = '' - else: - headerstr = '\nHEADERS:\n ' + '\n '.join([('%s: %s' % (k,v)) - for k,v in headers.items()]) - self.log.info('%s %s%s%s', method, url, formstr, headerstr) - ####### - - return self._http_request(url, method, body, headers) - - def _raise_service_error(self, url, status, ctype, body): - - is_jsbody = (ctype.endswith('javascript') - or ctype.endswith('json')) - if str(status) == '400' and is_jsbody: - r = self._loadjson(body) - msg = r.messages[0] - raise MetawebError(u'%s %s %r' % (msg.get('code',''), msg.message, msg.info)) - - raise MetawebError, 'request failed: %s: %r %r' % (url, status, body) - - def _httpreq_json(self, *args, **kws): - resp, body = self._httpreq(*args, **kws) - return self._loadjson(body) - - def _loadjson(self, json): - # TODO really this should be accomplished by hooking - # simplejson to create attrdicts instead of dicts. - def struct2attrdict(st): - """ - copy a json structure, turning all dicts into attrdicts. - - copying descends instances of dict and list, including subclasses. - """ - if isinstance(st, dict): - return attrdict([(k,struct2attrdict(v)) for k,v in st.items()]) - if isinstance(st, list): - return [struct2attrdict(li) for li in st] - return st - - if json == '': - self.log.error('the empty string is not valid json') - raise MetawebError('the empty string is not valid json') - - try: - r = simplejson.loads(json) - except ValueError, e: - self.log.error('error parsing json string %r' % json) - raise MetawebError, 'error parsing JSON string: %s' % e - - return struct2attrdict(r) - - def _check_mqlerror(self, r): - if r.code != '/api/status/ok': - for msg in r.messages: - self.log.error('mql error: %s %s %r' % (msg.code, msg.message, msg.get('query', None))) - raise MetawebError, 'query failed: %s\n%r' % (r.messages[0].code, r.messages[0].get('query', None)) - - def _mqlresult(self, r): - self._check_mqlerror(r) - - self.log.info('result: %s', Delayed(logformat, r)) - - return r.result - - - - def login(self, username=None, password=None): - """sign in to the service. For a more complete description, - see http://www.freebase.com/view/en/api_account_login""" - - service = '/api/account/login' - - username = username or self.username - password = password or self.password - - assert username is not None - assert password is not None - - self.log.debug('LOGIN USERNAME: %s', username) - - r = self._httpreq_json(service, 'POST', - form=dict(username=username, - password=password)) - - if r.code != '/api/status/ok': - raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) - - self.log.debug('LOGIN RESP: %r', r) - self.log.debug('LOGIN COOKIES: %s', self.cookiejar) - - def logout(self): - """logout of the service. For a more complete description, - see http://www.freebase.com/view/en/api_account_logout""" - - service = '/api/account/logout' - - self.log.debug("LOGOUT") - - r = self._httpreq_json(service, 'GET') - - if r.code != '/api/status/ok': - raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) #this should never happen - - def user_info(self, mql_output=None): - """ get user_info. For a more complete description, - see http://www.freebase.com/view/en/api_service_user_info""" - - service = "/api/service/user_info" - - qstr = simplejson.dumps(mql_output) - - r = self._httpreq_json(service, 'POST', form=dict(mql_output=qstr)) - return r - - def loggedin(self): - """check to see whether a user is logged in or not. For a - more complete description, see http://www.freebase.com/view/en/api_account_loggedin""" - - service = "/api/account/loggedin" - try: - r = self._httpreq_json(service, 'GET') - if r.code == "/api/status/ok": - return True - - except MetawebError, me: - return False - - def mqlreaditer(self, sq, asof=None): - """read a structure query.""" - - cursor = True - - while 1: - subq = dict(query=[sq], cursor=cursor, escape=False) - if asof: - subq['as_of_time'] = asof - - qstr = simplejson.dumps(subq) - - service = '/api/service/mqlread' - - r = self._httpreq_json(service, form=dict(query=qstr)) - - for item in self._mqlresult(r): - yield item - - if r['cursor']: - cursor = r['cursor'] - self.log.info('CONTINUING with %s', cursor) - else: - return - - def mqlread(self, sq, asof=None): - """read a structure query. For a more complete description, - see http://www.freebase.com/view/en/api_service_mqlread""" - subq = dict(query=sq, escape=False) - if asof: - subq['as_of_time'] = asof - - if isinstance(sq, list): - subq['cursor'] = True - - service = '/api/service/mqlread' - - self.log.info('%s: %s', - service, - Delayed(logformat, sq)) - - qstr = simplejson.dumps(subq) - r = self._httpreq_json(service, form=dict(query=qstr)) - - return self._mqlresult(r) - - def mqlreadmulti(self, queries, asof=None): - """read a structure query""" - keys = [('q%d' % i) for i,v in enumerate(queries)]; - envelope = {} - for i,sq in enumerate(queries): - subq = dict(query=sq, escape=False) - if asof: - subq['as_of_time'] = asof - - # XXX put this back once mqlreadmulti is working in general - #if isinstance(sq, list): - # subq['cursor'] = True - envelope[keys[i]] = subq - - service = '/api/service/mqlread' - - self.log.info('%s: %s', - service, - Delayed(logformat, envelope)) - - qstr = simplejson.dumps(envelope) - rs = self._httpreq_json(service, form=dict(queries=qstr)) - - self.log.info('%s result: %s', - service, - Delayed(simplejson.dumps, rs, indent=2)) - - return [self._mqlresult(rs[key]) for key in keys] - - def raw(self, id): - """translate blob from id. For a more complete description, - see http://www.freebase.com/view/en/api_trans_raw""" - url = '/api/trans/raw' + urlquote(id) - - self.log.info(url) - - resp, body = self._httpreq(url) - - self.log.info('raw is %d bytes' % len(body)) - - return body - - def blurb(self, id, break_paragraphs=False, maxlength=200): - """translate only the text in blob from id. For a more - complete description, see http://www.freebase.com/view/en/api_trans_blurb""" - url = '/api/trans/blurb' + urlquote(id) - - self.log.info(url) - - resp, body = self._httpreq(url, form=dict(break_paragraphs=break_paragraphs, maxlength=maxlength)) - - self.log.info('blurb is %d bytes' % len(body)) - - return body - - def image_thumb(self, id, maxwidth=None, maxheight=None, mode="fit", onfail=None): - """ given the id of an image, this will return a URL of a thumbnail of the image. - The full details of how the image is cropped and finessed is detailed at - http://www.freebase.com/view/en/api_trans_image_thumb """ - - service = "/api/trans/image_thumb" - assert mode in ["fit", "fill", "fillcrop", "fillcropmid"] - - form = dict(mode=mode) - if maxwidth is not None: - form["maxwidth"] = maxwidth - if maxheight is not None: - form["maxheight"] = maxheight - if onfail is not None: - form["onfail"] = onfail - - resp, body = self._httpreq(service + urlquote(id), form=form) - self.log.info('image is %d bytes' % len(body)) - - return body - - def mqlwrite(self, sq): - """do a mql write. For a more complete description, - see http://www.freebase.com/view/en/api_service_mqlwrite""" - query = dict(query=sq, escape=False) - qstr = simplejson.dumps(query) - - self.log.debug('MQLWRITE: %s', qstr) - - service = '/api/service/mqlwrite' - - self.log.info('%s: %s', - service, - Delayed(logformat,sq)) - - r = self._httpreq_json(service, 'POST', - form=dict(query=qstr)) - - self.log.debug('MQLWRITE RESP: %r', r) - return self._mqlresult(r) - - def mqlcheck(self, sq): - """ See if a write is valid, and see what would happen, but do not - actually do the write """ - - query = dict(query=sq, escape=False) - qstr = simplejson.dumps(query) - - self.log.debug('MQLCHECK: %s', qstr) - - service = '/api/service/mqlcheck' - - self.log.info('%s: %s', - service, - Delayed(logformat, sq)) - - r = self._httpreq_json(service, 'POST', - form=dict(query=qstr)) - - - self.log.debug('MQLCHECK RESP: %r', r) - - return self._mqlresult(r) - - def mqlflush(self): - """ask the service not to hand us old data""" - self.log.debug('MQLFLUSH') - - service = '/api/service/touch' - r = self._httpreq_json(service) - - self._check_mqlerror(r) - return True - - def touch(self): - """ make sure you are accessing the most recent data. For a more - complete description, see http://www.freebase.com/view/en/api_service_touch""" - return self.mqlflush() - - - def upload(self, body, content_type, document_id=False, permission_of=False): - """upload to the metaweb. For a more complete description, - see http://www.freebase.com/view/en/api_service_upload""" - - service = '/api/service/upload' - - self.log.info('POST %s: %s (%d bytes)', - service, content_type, len(body)) - - - headers = {} - if content_type is not None: - headers['content-type'] = content_type - - form = None - if document_id is not False: - if document_id is None: - form = { 'document': '' } - else: - form = { 'document': document_id } - if permission_of is not False: - if form: - form['permission_of'] = permission_of - else: - form = { 'permission_of' : permission_of } - - # note the use of both body and form. - # form parameters get encoded into the URL in this case - r = self._httpreq_json(service, 'POST', - headers=headers, body=body, form=form) - return self._mqlresult(r) - - - def version(self): - """ get versions for various parts of freebase. For a more - complete description, see http://www.freebase.com/view/en/api_version""" - - service = "/api/version" - r = self._httpreq_json(service) - - self._check_mqlerror(r) - return r - - ### DEPRECATED IN API - def reconcile(self, name, etype=['/common/topic']): - """DEPRECATED: reconcile name to guid. For a more complete description, - see http://www.freebase.com/view/en/dataserver_reconciliation""" - - service = '/dataserver/reconciliation' - r = self._httpreq_json(service, 'GET', form={'name':name, 'types':','.join(etype)}) - - - # TODO non-conforming service, fix later - #self._mqlresult(r) - return r - - -if __name__ == '__main__': - console = logging.StreamHandler() - console.setLevel(logging.DEBUG) - - mss = HTTPMetawebSession('sandbox.freebase.com') - - self.mss.log.setLevel(logging.DEBUG) - self.mss.log.addHandler(console) - - - print mss.mqlread([dict(name=None, type='/type/type')]) diff --git a/freebase-api/freebase/fcl/README.txt b/freebase-api/freebase/fcl/README.txt deleted file mode 100644 index 2e03d71..0000000 --- a/freebase-api/freebase/fcl/README.txt +++ /dev/null @@ -1,8 +0,0 @@ - -This is a command-line tool for performing various freebase commands. - -It should be installed by default as "fcl". - -Try "fcl help" after installing the freebase-api library. - - diff --git a/freebase-api/freebase/fcl/__init__.py b/freebase-api/freebase/fcl/__init__.py deleted file mode 100755 index e69de29..0000000 --- a/freebase-api/freebase/fcl/__init__.py +++ /dev/null diff --git a/freebase-api/freebase/fcl/commands.py b/freebase-api/freebase/fcl/commands.py deleted file mode 100755 index 0fdc283..0000000 --- a/freebase-api/freebase/fcl/commands.py +++ /dev/null @@ -1,688 +0,0 @@ -# ================================================================== -# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ==================================================================== - -import os, sys, re, time -from fbutil import * - -import simplejson - -import freebase.rison as rison -from freebase.api import attrdict - - -def cmd_help(fb, command=None): - """get help on commands - - %prog help [cmd] - """ - - if command is not None: - cmd = fb.commands[command] - print ' %s: %s' % (cmd.name, cmd.shortdoc) - print cmd.doc - return - - print """ -the interface to this tool is loosely modeled on the -"svn" command-line tool. many commands work slightly -differently from their unix and svn equivalents - -read the doc for the command first. - -use "%s help <subcommand>" for help on a particular -subcommand. -""" % fb.progpath - - fb.oparser.print_help() - - print 'available subcommands:' - cmds = sorted(fb.commands.keys()) - - for fk in cmds: - cmd = fb.commands[fk] - print ' %s: %s' % (cmd.name, cmd.shortdoc) - - -def cmd_wikihelp(fb): - """get help on commands in mediawiki markup format - - %prog wikihelp - """ - - print """usage: %s subcommand [ args ... ] - - the interface to this tool is loosely modeled on the - "svn" command-line tool. many commands work slightly - differently from their unix and svn equivalents - - read the doc for the command first. - """ % fb.progpath - - print 'subcommands:' - for fk in sorted(fb.commands.keys()): - cmd = fb.commands[fk] - - # mediawiki markup cares about the difference between - # a blank line and a blank line with a bunch of - # spaces at the start of it. ewww. - doc = re.sub(r'\r?\n {0,8}', '\n ', cmd.doc) - - print '==== %s %s: %s ====' % (fb.progpath, cmd.name, cmd.shortdoc) - print doc - print - - -# this command is disabled because it relies on some svn glue that -# was not sufficiently well thought out. -def old_cmd_pwid(fb): - """show the "current working namespace id" - - %prog pwid - - by default, relative ids can't be resolved. however, if you - run the fb tool from a directory that has the svn property - 'freebase:id' set, relative ids will be resolved relative - to that id. the idea is that you can create an svn tree - that parallels the freebase namespace tree. - - for example: - $ %prog pwid - - $ svn propset freebase:id "/freebase" . - - $ %prog pwid - /freebase - - """ - if fb.cwid is not None: - print fb.cwid - else: - print '' - - -def cmd_ls(fb, path=None): - """list the keys in a namespace - - %prog ls [id] - """ - path = fb.absid(path) - q = {'id': path, - '/type/namespace/keys': [{'value': None, - 'namespace': { - 'id': None, - 'type': [] - }, - 'optional':True - }] - } - - r = fb.mss.mqlread(q) - if r is None: - raise CmdException('query for id %r failed' % path) - - #sys.stdout.write(' '.join([mk.value for mk in r['/type/namespace/keys']])) - - for mk in r['/type/namespace/keys']: - print mk.value - - if 0: - suffix = '' - if ('/type/namespace' in mk.namespace.type - or '/type/domain' in mk.namespace.type): - suffix = '/' - print mk.value, mk.namespace.id+suffix - - -def cmd_mkdir(fb, path): - """create a new freebase namespace - %prog mkdir id - - create a new instance of /type/namespace at the given - point in id space. if id already exists, it should be - a namespace. - """ - path = fb.absid(path) - dir,file = dirsplit(path) - wq = { 'create': 'unless_exists', - 'key':{ - 'connect': 'insert', - 'namespace': dir, - 'value': file - }, - 'name': path, - 'type': '/type/namespace' - } - - r = fb.mss.mqlwrite(wq) - -def cmd_ln(fb, src, dst): - """create a namespace key - %prog ln srcid dstid - - create a new namespace link at dstid to the object - currently at srcid. - """ - src = fb.absid(src) - dst = fb.absid(dst) - dir,file = dirsplit(dst) - wq = { 'id': src, - 'key':{ - 'connect': 'insert', - 'namespace': dir, - 'value': file - } - } - - r = fb.mss.mqlwrite(wq) - - -def cmd_rm(fb, path): - """unlink a namespace key - %prog rm id - - remove the /type/key that connects the given id to its - parent. id must be a path for this to make any sense. - - note that is like unix 'unlink' rather than 'rm'. - it won't complain if the 'subdirectory' contains data, - since that data will still be accessible to other queries. - it's not like 'rm -rf' either, because it doesn't - disturb anything other than the one directory entry. - """ - path = fb.absid(path) - dir,file = dirsplit(path) - - wq = { 'id': path, - 'key':{ - 'connect': 'delete', - 'namespace': dir, - 'value': file - } - } - - r = fb.mss.mqlwrite(wq) - -def cmd_mv(fb, src, dst): - """rename srcid to dstid. - %prog mv srcid dstid - - equivalent to: - $ fb ln <srcid> <dstid> - $ fb rm <srcid> - """ - cmd_ln(fb, src, dst) - cmd_rm(fb, src) - -def cmd_cat(fb, id, include_headers=False): - """download a document from freebase to stdout - %prog cat id - - equivalent to "%prog get id -". - """ - return cmd_get(fb, id, localfile='-', include_headers=include_headers) - -def cmd_get(fb, id, localfile=None, include_headers=False): - """download a file from freebase - %prog get id [localfile] - - download the document or image with the given id from freebase - into localfile. localfile '-' means stdout. localfile - defaults to a file in the current directory with the same name - as the last key in the path, possibly followed by a metadata - extension like .html or .txt. - """ - id = fb.absid(id) - dir,file = dirsplit_unsafe(id) - - def read_content(id, content_only=False): - c = attrdict(id=id) - cq = { 'id': id, - 'type': [], - '/common/document/content': None, - '/common/document/source_uri': None, - '/type/content/media_type': { 'name':None, - 'optional': True }, - #'/type/content/text_encoding': { 'name':None }, - '/type/content/blob_id':None, - } - cd = fb.mss.mqlread(cq) - if '/type/content' in cd.type: - c.media_type = cd['/type/content/media_type'].name - #c.text_encoding = cd['/type/content/text_encoding'].name - c.sha256 = cd['/type/content/blob_id'] - return c - - if content_only: - raise CmdException('%s is not a content id' % id) - - cid = cd['/common/document/content'] - if cid is not None: - return read_content(cid, content_only=True) - - # in this case we don't have a content object - if cd['/common/document/source_uri'] is not None: - return None - - raise CmdException('%s is not a content or document id' % id) - - - content = read_content(id) - log.debug('COBJ %r' % content) - if content is not None: - fileext = media_type_to_extension.get(content.media_type, None) - else: - fileext = None - - if localfile == '-': - ofp = sys.stdout - else: - if localfile is None: - implicit_outfile = True - localfile = file - elif re.match(r'[/\\]$', localfile): - implicit_outfile = True - localfile = localfile + file - else: - implicit_outfile = False - localfile = os.path.abspath(localfile) - - # add file extension based on content-type: - # should be an option to disable this - if implicit_outfile and fileext is not None: - localfile += '.' + fileext - - # if we didn't explicitly name the output file, - # don't destroy an existing file - localfile_base = localfile - count = 0 - while implicit_outfile and os.path.exists(localfile): - count += 1 - localfile = '%s.%d' % (localfile_base, count) - ofp = open(localfile, 'wb') - - body = fb.mss.trans(id) - - if include_headers: - # XXX show content-type, what else? - pass - - ofp.write(body) - - if localfile != '-': - print ('%s saved (%d bytes)' % (localfile, len(body))) - ofp.close() - - -def cmd_put(fb, localfile, id=None, content_type=None): - """upload a document to freebase -- EXPERIMENTAL - %prog put localfile [id] [content-type] - - upload the document or image in localfile to given freebase - id. if localfile is '-' the data will be read from stdin. - - if id is missing or empty, a new document will be created. - later the id might default to something computed from localfile - and any svn attributes it has. - - output: a single line, the id of the document. - """ - if content_type is None: - ext = re.sub('^.*\.([^/.]+)$', r'\1', localfile) - media_type = extension_to_media_type.get(ext, None) - - if media_type is None: - raise CmdException('could not infer a media type from extension %r: please specify it' - % ext) - - if media_type.startswith('text/'): - # this is a bad assumption. should sniff it? - text_encoding = 'utf-8' - content_type = '%s;charset=%s' % (media_type, text_encoding) - else: - content_type = media_type - - new_id = None - if id is not None: - idinfo = fb.mss.mqlread({ 'id': id, 'type': '/common/document' }) - if idinfo is None: - new_id = id - id = None - - body = open(localfile, 'rb').read() - r = fb.mss.upload(body, content_type, document_id=id) - - if new_id is None: - print r.document - else: - cmd_ln(fb, r.document, new_id) - print new_id - - -def cmd_dump(fb, id): - """show all properties of a freebase object - %prog dump object_id - """ - id = fb.absid(id) - - import inspect - - r = inspect.inspect_object(fb.mss, id) - if r is None: - raise CmdException('no match for id %r' % id) - - for k in sorted(r.keys()): - vs = r[k] - for v in vs: - id = v.get('id', '') - - name = '%r' % (v.get('name') or v.get('value')) - if name == 'None': name = '' - - type = v.get('type', '') - if type == '/type/text': - extra = v.get('lang', '') - elif type == '/type/key': - extra = v.get('namespace', '') - else: - extra = '' - - fb.trow(k, id, name, type, extra) - - -def cmd_pget(fb, id, propid): - """get a property of a freebase object -- EXPERIMENTAL - %prog pget object_id property_id - - get the property named by property_id from the object. - - XXX output quoting is not well specified. - - property_id must be a fully qualified id for now. - - prints one line for each match. - - if propid ends in '*' this does a wildcard for a particular type. - """ - id = fb.absid(id) - proptype, propkey = dirsplit(propid) - - if propkey != '*': - # look up the prop - q = { 'id': id, - propid: [{}], - } - r = fb.mss.mqlread(q) - for v in r[propid]: - if 'value' in v: - print v.value - else: - print v.id - - else: - # look up the prop - q = { 'id': id, - '*': [{}], - } - if isinstance(proptype, basestring): - q['type'] = proptype - - r = fb.mss.mqlread(q) - - for k in sorted(r.keys()): - v = r[k]; - if 'value' in v: - print '%s %s' % (k, v.value) - else: - print '%s %s' % (k, v.id) - -def cmd_pdel(fb, id, propid, oldval): - """delete a property of a freebase object -- EXPERIMENTAL - %prog pdel object_id property_id oldvalue - - set the property named by property_id on the object. - value is an id or a json value. XXX this is ambiguous. - - property_id must be a fully qualified id for now. - - for now you need to provide a "oldval" argument, - later this tool will query and perhaps prompt if the - deletion is ambiguous. - - prints a single line, either 'deleted' or 'missing' - """ - return cmd_pset(fb, id, propid, None, oldval) - - -def cmd_touch(fb): - """bypass any cached query results the service may have. use sparingly. - """ - fb.mss.mqlflush() - - -def cmd_pset(fb, id, propkey, val, oldval=None, extra=None): - """set a property of a freebase object -- EXPERIMENTAL - %prog pset object_id property_id value - - set the property named by property_id on the object. - value is an id or a json value. XXX this is ambiguous. - - property_id must be a fully qualified id for now. - - if the property should be a unique property, this will - write with 'connect:update'. if the property may have - multiple, it is written with 'connect:insert'. - - prints a single line, either 'inserted' or 'present' - """ - id = fb.absid(id) - - propid = fb.absprop(propkey) - - # look up the prop - pq = { 'id': propid, - 'type': '/type/property', - 'name': None, - 'unique': None, - 'expected_type': { - 'id': None, - 'name': None, - 'default_property': None, - 'optional': True, - }, - } - - prop = fb.mss.mqlread(pq) - - if prop is None: - raise CmdException('can\'t resolve property key %r - use an absolute id' % propid); - - if propid.startswith('/type/object/') or propid.startswith('/type/value/'): - propkey = re.sub('/type/[^/]+/', '', propid); - else: - propkey = propid - - wq = { 'id': id, - propkey: { - } - } - - if val is None: - val = oldval - wq[propkey]['connect'] = 'delete' - elif prop.unique: - wq[propkey]['connect'] = 'update' - else: - wq[propkey]['connect'] = 'insert' - - if prop.expected_type is None: - wq[propkey]['id'] = val - elif prop.expected_type.id not in value_types: - wq[propkey]['id'] = val - else: - wq[propkey]['value'] = val - - if prop.expected_type.id == '/type/text': - if extra is not None: - lang = extra - else: - lang = '/lang/en' - wq[propkey]['lang'] = lang - - if prop.expected_type.id == '/type/key': - if extra is not None: - wq[propkey]['namespace'] = extra - else: - raise CmdException('must specify a namespace to pset /type/key') - - r = fb.mss.mqlwrite(wq) - print r[propkey]['connect'] - -def cmd_login(fb, username=None, password=None): - """login to the freebase service - %prog login [username [password]] - - cookies are maintained in a file so - they are available to the next invocation. - prompts for username and password if not given - """ - import getpass - if username is None: - sys.stdout.write('freebase.com username: ') - username = sys.stdin.readline() - if not username: - raise CmdException('usernmae required for login') - username = re.compile('\n$').sub('', username) - if password is None: - password = getpass.getpass('freebase.com password: ') - - fb.mss.username = username - fb.mss.password = password - fb.mss.login() - -def cmd_logout(fb): - """logout from the freebase service - %prog logout - - deletes the login cookies - """ - fb.cookiejar.clear(domain=fb.service_host.split(':')[0]) - - -def cmd_find(fb, qstr): - """print all ids matching a given constraint. - - if the query string starts with "{" it is treated as json. - otherwise it is treated as o-rison. - - %prog find - """ - if qstr.startswith('{'): - q = simplejson.loads(qstr) - else: - q = rison.loads('(' + qstr + ')') - - if 'id' not in q: - q['id'] = None - - results = fb.mss.mqlreaditer(q) - for r in results: - print r.id - -def cmd_q(fb, qstr): - """run a freebase query. - - if the query string starts with "{" it is treated as json. - otherwise it is treated as o-rison. - - dump the result as json. - - %prog q - """ - if qstr.startswith('{'): - q = simplejson.loads(qstr) - else: - q = rison.loads('(' + qstr + ')') - - # results could be streamed with a little more work - results = fb.mss.mqlreaditer(q) - print simplejson.dumps(list(results), indent=2) - -def cmd_open(fb, id): - """open a web browser on the given id. works on OSX only for now. - - %prog open /some/id - """ - os.system("open 'http://www.freebase.com/view%s'" % id) - - -def cmd_log(fb, id): - """log changes pertaining to a given id. - - INCOMPLETE - - %prog log /some/id - """ - - null = None - true = True - false = False - - baseq = { - 'type': '/type/link', - 'source': null, - 'master_property': null, - 'attribution': null, - 'timestamp': null, - 'operation': null, - 'valid': null, - 'sort': '-timestamp' - }; - - queries = [ - { - 'target_value': { '*': null }, - 'target': { 'id': null, 'name': null, 'optional': true }, - }, - { - 'target': { 'id': null, 'name': null }, - }] - - for i,q in list(enumerate(queries)): - q.update(baseq) - queries[i] = [q] - - - valuesfrom,linksfrom = fb.mss.mqlreadmulti(queries) - - for link in linksfrom: - # fb.trow(link.master_property.id, ...) - print simplejson.dumps(link, indent=2) - - for link in valuesfrom: - # fb.trow(link.master_property.id, ...) - print simplejson.dumps(link, indent=2) - - diff --git a/freebase-api/freebase/fcl/fbutil.py b/freebase-api/freebase/fcl/fbutil.py deleted file mode 100755 index ed1e511..0000000 --- a/freebase-api/freebase/fcl/fbutil.py +++ /dev/null @@ -1,113 +0,0 @@ -# ================================================================== -# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ==================================================================== - - -import re - -import logging -log = logging.getLogger() - - -class FbException(Exception): - pass - -class CmdException(Exception): - pass - -media_types = { - 'html': ['text/html'], - 'txt': ['text/plain'], - - 'xml': ['text/xml', - 'application/xml'], - - 'atom': ['application/atom+xml'], - - 'js': ['text/javascript', - 'application/javascript', - 'application/x-javascript'], - - 'json': ['application/json'], - - 'jpg': ['image/jpeg', - 'image/pjpeg'], - - 'gif': ['image/gif'], - - 'png': ['image/png'], - } - -extension_to_media_type = dict([(k,vs[0]) for k,vs in media_types.items()]) -media_type_to_extension = {} -for k,vs in media_types.items(): - for v in vs: - media_type_to_extension[v] = k - - - - -DIRSPLIT = re.compile(r'^(.+)/([^/]+)$') - -def dirsplit_unsafe(id): - m = DIRSPLIT.match(id) - if m is None: - return (None, id) - dir,file = m.groups() - return (dir,file) - -def dirsplit(id): - dir,file = dirsplit_unsafe(id) - if dir == '/guid': - raise FbException('%r is not a freebase keypath' % (id,)) - return (dir,file) - -value_types = [ - '/type/text', - '/type/key', - '/type/rawstring', - '/type/float', - '/type/int', - '/type/boolean', - '/type/uri', - '/type/datetime', - '/type/id', - '/type/enumeration', - ] - -default_propkeys = { - 'value': '/type/value/value', - 'id': '/type/object/id', - 'guid': '/type/object/guid', - 'type': '/type/object/type', - 'name': '/type/object/name', - 'key': '/type/object/key', - 'timestamp': '/type/object/timestamp', - 'permission': '/type/object/permission', - 'creator': '/type/object/creator', - 'attribution': '/type/object/attribution' -}; diff --git a/freebase-api/freebase/fcl/fcl.py b/freebase-api/freebase/fcl/fcl.py deleted file mode 100755 index a0299b8..0000000 --- a/freebase-api/freebase/fcl/fcl.py +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/env python -# ================================================================== -# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ==================================================================== - -import os, sys, re, time, stat - -from optparse import OptionParser -import getpass -import cookielib -import logging -import simplejson - - -from fbutil import FbException, CmdException, log, default_propkeys -console = logging.StreamHandler() -log.addHandler(console) - -from freebase.api import HTTPMetawebSession, MetawebError, attrdict - -_cookiedir = None -if os.environ.has_key('HOME'): - _cookiedir = os.path.join(os.environ['HOME'], '.pyfreebase') - - -class Command(object): - def __init__(self, module, name, func): - self.module = module - self.name = name - self.func = func - - PROG = re.compile(r'\%prog') - NEWLINE = re.compile(r'(?:\r?\n)+') - - # fill in self.doc - if isinstance(func.__doc__, basestring): - doc = PROG.sub('fcl', func.__doc__ + '\n') - self.shortdoc, self.doc = NEWLINE.split(doc, 1) - else: - self.shortdoc = '(missing documentation)' - self.doc = '(missing documentation)' - - - -class FbCommandHandler(object): - - def __init__(self): - self.service_host = 'www.freebase.com' - self.cookiejar = None - self.cwid = '' - self.progpath = 'fcl' - self.commands = {} - - self.cookiefile = None - if _cookiedir is not None: - self.cookiefile = os.path.join(_cookiedir, 'cookiejar') - - - def init(self): - if self.cookiefile is not None: - self.cookiejar = cookielib.LWPCookieJar(self.cookiefile) - if os.path.exists(self.cookiefile): - try: - self.cookiejar.load(ignore_discard=True) - except cookielib.LoadError: - log.warn('error loading cookies') - - #print 'start cookies %r' % self.cookiejar - - self.mss = HTTPMetawebSession(self.service_host, - cookiejar=self.cookiejar) - - - def absid(self, path): - if path is None: - path = '' - if path.startswith('/'): - return path - - if not isinstance(self.cwid, basestring) or not self.cwid.startswith('/'): - # svn cwid support is disabled because it relies on some svn glue that - # was not sufficiently well thought out. - # raise CmdException("can't resolve relative id %r without cwid - see 'fcl help pwid'" % (path)) - raise CmdException("no support for relative id %r" % (path)) - - if path == '' or path == '.': - return self.cwid - - return self.cwid + '/' + path - - - def absprop(self, propkey): - if propkey.startswith('/'): - return propkey - - # check schemas of /type/object and /type/value, - # as well as other reserved names - if propkey in default_propkeys: - return default_propkeys[propkey] - - return self.absid(propkey) - - def thead(self, *args): - strs = ['%r' % arg - for arg in args] - print '\t'.join(strs) - - def trow(self, *args): - print '\t'.join(args) - return - strs = ['%r' % arg - for arg in args] - print '\t'.join(strs) - - def save(self): - #print 'end cookies %r' % self.cookiejar - if _cookiedir and self.cookiefile.startswith(_cookiedir): - # create private cookiedir if needed - if not os.path.exists(_cookiedir): - os.mkdir(_cookiedir, 0700) - os.chmod(_cookiedir, stat.S_IRWXU) - - if self.cookiejar is None: - return - - self.cookiejar.save(ignore_discard=True) - - # save the cwd and other state too - - - def import_commands(self, modname): - """ - import new fb commands from a file - """ - namespace = {} - - pyimport = 'from %s import *' % modname - exec pyimport in namespace - mod = sys.modules.get(modname) - - commands = [Command(mod, k[4:], getattr(mod, k)) - for k in getattr(mod, '__all__', dir(mod)) - if (k.startswith('cmd_') - and callable(getattr(mod, k)))] - - - for cmd in commands: - log.info('importing %r' % ((cmd.name, cmd.func),)) - self.commands[cmd.name] = cmd - - log.info('imported %r from %r' % (modname, mod.__file__)) - - - def dispatch(self, cmd, args): - if cmd in self.commands: - try: - self.commands[cmd].func(self, *args) - except KeyboardInterrupt, e: - sys.stderr.write('%s\n' % (str(e),)) - except FbException, e: - sys.stderr.write('%s\n' % (str(e),)) - except CmdException, e: - sys.stderr.write('%s\n' % (str(e),)) - except MetawebError, e: - sys.stderr.write('%s\n' % (str(e),)) - else: - self.oparser.error('unknown subcommand %r, try "%s help"' % (cmd, self.progpath)) - - self.save() - - - def cmdline_main(self): - op = OptionParser(usage='%prog [options] command [args...] ') - self.oparser = op - - op.disable_interspersed_args() - - op.add_option('-d', '--debug', dest='debug', - default=False, action='store_true', - help='turn on debugging output') - - op.add_option('-v', '--verbose', dest='verbose', - default=False, action='store_true', - help='verbose output') - - op.add_option('-V', '--very-verbose', dest='very_verbose', - default=False, action='store_true', - help='lots of debug output') - - op.add_option('-s', '--service', dest='service_host', - metavar='HOST', - default=self.service_host, - help='Freebase HTTP service address:port') - - op.add_option('-S', '--sandbox', dest='use_sandbox', - default=False, action='store_true', - help='shortcut for --service=sandbox.freebase.com') - - op.add_option('-c', '--cookiejar', dest='cookiefile', - metavar='FILE', - default=self.cookiefile, - help='Cookie storage file (will be created if missing)') - options,args = op.parse_args() - - if len(args) < 1: - op.error('required subcommand missing') - - - loglevel = logging.WARNING - if options.verbose: - loglevel = logging.INFO - if options.very_verbose: - loglevel = logging.DEBUG - - console.setLevel(loglevel) - log.setLevel(loglevel) - - if options.use_sandbox: - self.service_host = 'sandbox.freebase.com' - else: - self.service_host = options.service_host - - self.cookiefile = options.cookiefile - #self.progpath = sys.argv[0] - - self.init() - - self.mss.log.setLevel(loglevel) - self.mss.log.addHandler(console) - - self.import_commands('freebase.fcl.commands') - self.import_commands('freebase.fcl.mktype') - - cmd = args.pop(0) - self.dispatch(cmd, args) - -# entry point for script -def main(): - try: - # turn off crlf output on windows so we work properly - # with unix tools. - import msvcrt - msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) - except ImportError: - pass - - fb = FbCommandHandler() - fb.cmdline_main() - -if __name__ == '__main__': - main() diff --git a/freebase-api/freebase/fcl/inspect.py b/freebase-api/freebase/fcl/inspect.py deleted file mode 100755 index 8cd04d2..0000000 --- a/freebase-api/freebase/fcl/inspect.py +++ /dev/null @@ -1,218 +0,0 @@ -# ================================================================== -# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ==================================================================== - - -# -# -# wrap all the nastiness needed for a general mql inspect query -# -# - -import os, sys, re - - -null = None -true = True -false = False - -inspect_query = { - 'name': null, - 'type': [], - - '/type/reflect/any_master': [{ - 'optional':true, - 'id': null, - 'name': null, - 'link': { - 'master_property': { - 'id': null, - 'schema': null - } - } - }], - - '/type/reflect/any_reverse': [{ - 'optional':true, - 'id': null, - 'name': null, - 'link': { - 'master_property': { - 'id':null, - 'schema': null, - 'expected_type': null, - 'reverse_property': { - 'id': null, - 'schema': null, - 'optional': true - } - } - } - }], - - '/type/reflect/any_value': [{ - 'optional':true, - 'value': null, - 'link': { - 'master_property': { - 'id':null, - 'schema': null, - 'expected_type': null - }, - } - }], - - 't:/type/reflect/any_value': [{ - 'optional':true, - 'type': '/type/text', - 'value': null, - 'lang': null, - 'link': { - 'master_property': { - 'id':null, - 'schema': null - }, - } - }], - - '/type/object/creator': [{ - 'optional':true, - 'id':null, - 'name':null - }], - '/type/object/timestamp': [{ - 'optional':true, - 'value': null, - }], - - '/type/object/key': [{ - 'optional':true, - 'value': null, - 'namespace': null - }], - '/type/namespace/keys': [{ - 'optional':true, - 'value': null, - 'namespace': null - }] -} - - -def transform_result(result): - proptypes = {} - props = {} - - # copy a property from a /type/reflect clause - def pushtype(propdesc, prop): - tid = propdesc['schema'] - propid = propdesc['id'] - - if isinstance(prop, dict): - prop = dict(prop) - if 'link' in prop: - prop.pop('link') - - if tid not in proptypes: - proptypes[tid] = {} - if propid not in proptypes[tid]: - proptypes[tid][propid] = [] - - if propid not in props: - props[propid] = [] - props[propid].append(prop) - - # copy a property that isn't enumerated by /type/reflect - def pushprop(propid): - ps = result[propid] - if ps is None: - return - - # hack to infer the schema from id, not always reliable! - schema = re.sub(r'/[^/]+$', '', propid) - keyprop = dict(id=propid, schema=schema) - for p in ps: - pushtype(keyprop, p) - - ps = result['/type/reflect/any_master'] or [] - for p in ps: - propdesc = p.link.master_property - pushtype(propdesc, p) - - # non-text non-key values - ps = result['/type/reflect/any_value'] or [] - for p in ps: - propdesc = p.link.master_property - - # /type/text values are queried specially - # so that we can get the lang, so ignore - # them here. - if propdesc.expected_type == '/type/text': - continue - - pushtype(propdesc, p) - - # text values - ps = result['t:/type/reflect/any_value'] or [] - for p in ps: - propdesc = p.link.master_property - pushtype(propdesc, p) - - pushprop('/type/object/creator') - pushprop('/type/object/timestamp') - pushprop('/type/object/key') - pushprop('/type/namespace/keys') - - # now the reverse properties - ps = result['/type/reflect/any_reverse'] or [] - for prop in ps: - propdesc = prop.link.master_property.reverse_property - - # synthetic property descriptor for the reverse of - # a property with no reverse descriptor. - # note the bogus id starting with '-'. - if propdesc is None: - # schema = prop.link.master_property.expected_type - # if schema is None: - # schema = 'other' - - schema = 'other' - propdesc = dict(id='-' + prop.link.master_property.id, - schema=schema) - - pushtype(propdesc, prop) - - #return proptypes - return props - - -def inspect_object(mss, id): - q = dict(inspect_query) - q['id'] = id - r = mss.mqlread(q) - if r is None: - return None - return transform_result(r) diff --git a/freebase-api/freebase/fcl/mktype.py b/freebase-api/freebase/fcl/mktype.py deleted file mode 100755 index 30f8df7..0000000 --- a/freebase-api/freebase/fcl/mktype.py +++ /dev/null @@ -1,179 +0,0 @@ -# ================================================================== -# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ==================================================================== - -import os, sys, re, time -from fbutil import * - - -def cmd_mkobj(fb, id, typeid='/common/topic', name=''): - """create a new object with a given type -- EXPERIMENTAL - %prog mkobj new_id typeid name - - create a new object with type typeid at the given - namespace location. - - if present, name gives the display name of the new object. - - """ - id = fb.absid(id) - nsid, key = dirsplit(id) - - typeid = fb.absid(typeid) - - if name == '': - name = key - - wq = { 'create': 'unless_exists', - 'id': None, - 'name': name, - 'type': typeid, - 'key':{ - 'namespace': nsid, - 'value': key - }, - } - - # TODO add included types - - r = fb.mss.mqlwrite(wq) - print r.id,r.create - -def cmd_mktype(fb, id, name=''): - """create a new type -- EXPERIMENTAL - %prog mktype new_id name - - create a new object with type Type at the given - namespace location. - - this doesn't create any type hints. - - if present, name gives the display name of the new property - """ - id = fb.absid(id) - - nsid, key = dirsplit(id) - - if name == '': - name = key - - wq = { 'create': 'unless_exists', - 'id': None, - 'name': name, - 'type': '/type/type', - 'key':{ - 'namespace': nsid, - 'value': key - }, - } - - r = fb.mss.mqlwrite(wq) - print r.id,r.create - -def mkprop(fb, typeid, key, name='', vtype=None, master_property=None): - """helper to create a new property - """ - if name == '': - name = key - - wq = { 'create': 'unless_exists', - 'id': None, - 'type': '/type/property', - 'name': name, - 'schema': typeid, - 'key': { - 'namespace': typeid, - 'value': key - } - } - - if vtype is not None: - wq['expected_type'] = vtype - if master_property is not None: - wq['master_property'] = master_property - - return fb.mss.mqlwrite(wq) - - -def cmd_mkprop(fb, id, name='', vtype=None, revkey=None, revname=''): - """create a new property -- EXPERIMENTAL - %prog mkprop new_id [name] [expected_type] [reverse_property] [reverse_name] - - create a new object with type Property at the given - location. creates both the "schema" and "key" links - for the property, but doesn't create any freebase property - hints. - - if present, name gives the display name of the new property - """ - id = fb.absid(id) - if vtype is not None: - vtype = fb.absid(vtype) - - typeid, key = dirsplit(id) - - - r = mkprop(fb, typeid, key, name, vtype) - - # write the reverse property if specified - - print r.id, r.create - - if revkey is None: - return - - assert vtype is not None - - rr = mkprop(fb, vtype, revkey, revname, typeid, id) - print rr.id, rr.create - - -def cmd_publish_type(fb, typeid): - """try to publish a freebase type for the client - %prog publish_type typeid - - set /freebase/type_profile/published to the /freebase/type_status - instance named 'Published' - - - should also try to set the domain to some namespace that - has type:/type/domain - - """ - id = fb.absid(typeid) - - w = { - 'id': id, - '/freebase/type_profile/published': { - 'connect': 'insert', - 'type': '/freebase/type_status', - 'name': 'Published' - } - } - r = fb.mss.mqlwrite(w) - - print r['/freebase/type_profile/published']['connect'] diff --git a/freebase-api/freebase/rison.py b/freebase-api/freebase/rison.py deleted file mode 100755 index 2f4eebc..0000000 --- a/freebase-api/freebase/rison.py +++ /dev/null @@ -1,308 +0,0 @@ -# -# rison for python (parser only so far) -# see http://mjtemplate.org/examples/rison.html for more info -# - -###################################################################### -# -# the rison parser is based on javascript openlaszlo-json: -# Author: Oliver Steele -# Copyright: Copyright 2006 Oliver Steele. All rights reserved. -# Homepage: http:#osteele.com/sources/openlaszlo/json -# License: MIT License. -# Version: 1.0 -# - -# hacked by nix for use in uris -# ported to python by nix -# -# TODO -# -# switch to unicode -# fall through to simplejson if first char is not in '!(' - -# this allows code to use just one parser -# - - -import os, sys, re -#import simplejson -simplejson = None - -class ParserException(Exception): - pass - -class Parser(object): - WHITESPACE = '' - #WHITESPACE = " \t\n\r\f" - - # we divide the uri-safe glyphs into three sets - # <rison> and <reserved> classes are illegal in ids. - # <rison> - used by rison (possibly later) - # <reserved> - not common in strings, reserved - #not_idchar = "'!=:(),*@$;&"; - - idchar_punctuation = '_-./~' - not_idchar = ''.join([c for c in (chr(i) for i in range(127)) - if not (c.isalnum() - or c in idchar_punctuation)]) - - # additionally, we need to distinguish ids and numbers by first char - not_idstart = "-0123456789"; - - # regexp string matching a valid id - idrx = ('[^' + not_idstart + not_idchar + - '][^' + not_idchar + ']*') - - # regexp to check for valid rison ids - id_ok_re = re.compile('^' + idrx + '$', re.M) - - # regexp to find the end of an id when parsing - next_id_re = re.compile(idrx, re.M) - - def parse_json(self, str): - if len(str) > 0 and str[0] not in '!(': - return simplejson.loads(str) - return self.parse(str) - - def parse(self, str): - self.string = str - self.index = 0 - - value = self.readValue() - if self.next(): - raise ParserException("unable to parse rison string %r" % (str,)) - return value - - def readValue(self): - c = self.next() - - if c == '!': - return self.parse_bang() - if c == '(': - return self.parse_open_paren() - if c == "'": - return self.parse_single_quote() - if c in '-0123456789': - return self.parse_number() - - # fell through table, parse as an id - s = self.string - i = self.index-1 - - m = self.next_id_re.match(s, i) - if m: - id = m.group(0) - self.index = i + len(id) - return id # a string - - if c: - raise ParserException("invalid character: '" + c + "'") - raise ParserException("empty expression") - - def parse_array(self): - ar = [] - while 1: - c = self.next() - if c == ')': - return ar - - if c is None: - raise ParserException("unmatched '!('") - - if len(ar): - if c != ',': - raise ParserException("missing ','") - elif c == ',': - raise ParserException("extra ','") - else: - self.index -= 1 - n = self.readValue() - ar.append(n) - - return ar - - def parse_bang (self): - s = self.string - c = s[self.index] - self.index += 1 - if c is None: - raise ParserException('"!" at end of input') - if c not in self.bangs: - raise ParserException('unknown literal: "!' + c + '"') - x = self.bangs[c] - if callable(x): - return x(self) - - return x - - - def parse_open_paren (self): - count = 0 - o = {} - - while 1: - c = self.next() - if c == ')': - return o - if count: - if c != ',': - raise ParserException("missing ','") - elif c == ',': - raise ParserException("extra ','") - else: - self.index -= 1 - k = self.readValue() - - if self.next() != ':': - raise ParserException("missing ':'") - v = self.readValue() - - o[k] = v - count += 1 - - - def parse_single_quote(self): - s = self.string - i = self.index - start = i - segments = [] - - while 1: - if i >= len(s): - raise ParserException('unmatched "\'"') - - c = s[i] - i += 1 - if c == "'": - break - - if c == '!': - if start < i-1: - segments.append(s[start:i-1]) - c = s[i] - i += 1 - if c in "!'": - segments.append(c) - else: - raise ParserException('invalid string escape: "!'+c+'"') - - start = i - - - if start < i-1: - segments.append(s[start:i-1]) - self.index = i - return ''.join(segments) - - - # Also any number start (digit or '-') - def parse_number(self): - s = self.string - i = self.index - start = i-1 - state = 'int' - permittedSigns = '-' - transitions = { - 'int+.': 'frac', - 'int+e': 'exp', - 'frac+e': 'exp' - } - - while 1: - if i >= len(s): - i += 1 - break - - c = s[i] - i += 1 - - if '0' <= c and c <= '9': - continue - - if permittedSigns.find(c) >= 0: - permittedSigns = '' - continue - - state = transitions.get(state + '+' + c.lower(), None) - if state is None: - break - if state == 'exp': - permittedSigns = '-' - - self.index = i - 1 - s = s[start:self.index] - if s == '-': - raise ParserException("invalid number") - if re.search('[.e]', s): - return float(s) - return int(s) - - # return the next non-whitespace character, or undefined - def next(self): - l = len(self.string) - s = self.string - i = self.index - - while 1: - if i == len(s): - return None - c = s[i] - i += 1 - if c not in self.WHITESPACE: - break - - self.index = i - return c - - - bangs = { - 't': True, - 'f': False, - 'n': None, - '(': parse_array - } - - -def loads(s): - return Parser().parse(s) - -if __name__ == '__main__': - p = Parser() - - rison_examples = [ - "(a:0,b:1)", - "(a:0,b:foo,c:'23skidoo')", - "!t", - "!f", - "!n", - "''", - "0", - "1.5", - "-3", - "1e30", - "1e-30", - "G.", - "a", - "'0a'", - "'abc def'", - "()", - "(a:0)", - "(id:!n,type:/common/document)", - "!()", - "!(!t,!f,!n,'')", - "'-h'", - "a-z", - "'wow!!'", - "domain.com", - "'user@domain.com'", - "'US $10'", - "'can!'t'", - ]; - - for s in rison_examples: - print - print '*'*70 - print - print s - - print '%r' % (p.parse(s),) diff --git a/freebase-api/freebase/sandbox.py b/freebase-api/freebase/sandbox.py deleted file mode 100644 index 2d11be2..0000000 --- a/freebase-api/freebase/sandbox.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/python -# ======================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ======================================================================== - -import sys -from freebase.api.session import HTTPMetawebSession - -base = HTTPMetawebSession("sandbox.freebase.com") - -__all__ = ["HTTPMetawebSession"] - -# we want to add base's functions to __init__.py -# so that we can say freebase.func() and really -# just call base.func() - -# a little trick to refer to ourselves -self = sys.modules[__name__] - -for funcname in dir(base): - - # we only want the 'real' functions - if not funcname.startswith("_"): - func = getattr(base, funcname) - - # let's make sure we're getting functions - # instead of constants or whatever - if callable(func): - - # we're setting these functions - # so that they can be called like - # freebase.funcname -> base.func() - setattr(self, funcname, func) - - # make sure we import the base's - # functions if we import freebase - __all__.append(funcname) - -# we don't want any self-referencing -# business going. Plus, this is cleaner. -del self diff --git a/freebase-api/freebase/uritemplate.py b/freebase-api/freebase/uritemplate.py deleted file mode 100755 index 0a81147..0000000 --- a/freebase-api/freebase/uritemplate.py +++ /dev/null @@ -1,143 +0,0 @@ -# ======================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ======================================================================== - -# -# URI Templating in Python -# -# see http://bitworking.org/projects/URI-Templates/ -# and http://bitworking.org/news/URI_Templates -# -# note that this implementation may go away soon in -# favor of joe gregorio's own code: -# http://code.google.com/p/uri-templates/ -# -# -# this implementation can also parse URIs, as long as the -# template is sufficiently specific. to allow this to work -# the '/' character is forbidden in keys when parsing. -# later it should be possible to loosen this restriction. -# -# -# example: -# from whatever.uritemplate import expand_uri_template -# expand_uri_template('http://{host}/{file}', -# dict(host='example.org', -# file='fred')) -# -# TODO: -# allow parsing to be aware of http://www. and trailing / -# nail down quoting issues -# - -import os, sys, re -import urllib - -__all__ = ['expand_uri_template', 'URITemplate'] - - -def expand_uri_template(template, args): - """Expand a URI template using the given args dictionary. - """ - return URITemplate(template).run(args) - -def _uri_encode_var(v): - return urllib.quote(v, safe="-_.~!$&'()*+,;=:/?[]#@") - - -class URITemplate(object): - """a URITemplate is a URI with simple variable substitution. - """ - - VARREF = re.compile(r'\{([0-9a-zA-Z_]+)\}') - - def __init__(self, s): - """Compile a URITemplate from a string. - """ - self.template = s; - - self.params = [] - tsplit = self.VARREF.split(s) - rxs = ['^'] - for i in range(len(tsplit)): - if i % 2: - # track the vars used - self.params.append(tsplit[i]) - # vars match any string - # vars are assumed to lack '/' - this is imperfect... - rxs.append('([^/]*)') - else: - # quote special chars regexp interpretation - rxs.append(re.escape(tsplit[i])) - rxs.append('$') - self._parser = re.compile(''.join(rxs)) - - def __repr__(self): - return '<URITemplate %r>' % self.template - - def run (self, args): - """Expand the template using the given args. - """ - def repl(m): - key = m.group(1) - return _uri_encode_var(args.get(key, '')) - uri = self.VARREF.sub(repl,self.template) - - #if self.parse(uri) is None: - # print 're-parsing generated uri failed: %r, %r' % (uri, self.template) - return uri - - - def parse(self, uri): - """Try to parse a URI, extracting variable values. - """ - m = self._parser.match(uri) - if m is None: - return None - return dict(zip(self.params, m.groups())) - - -if __name__ == '__main__': - # - # testcases are imported from the URI::Template module on CPAN - # - import urllib2, simplejson - fp = urllib2.urlopen('http://search.cpan.org/src/BRICAS/URI-Template-0.09/t/data/spec.json') - testsuite = simplejson.loads(fp.read()) - vars = dict([(k.encode('utf-8'),v) for k,v in testsuite['variables'].items()]) - nsucceed = 0 - nfail = 0 - for test in testsuite['tests']: - ut = URITemplate(test['template']) - uri = ut.run(vars) - if uri != test['expected']: - print 'FAILED %r expected %r' % (uri, test['expected']) - print ' vars: %r' % (vars,) - nfail += 1 - else: - nsucceed += 1 - print 'tests: %d succeeded, %d failed' % (nsucceed, nfail) |