From c8e799acf3c1aae32bb51ab88ba54bd8d27a08cb Mon Sep 17 00:00:00 2001 From: nitromaster101 Date: Thu, 18 Jun 2009 23:43:53 +0000 Subject: finish uploading appengine example: had to force freebase... it was already in svn git-svn-id: http://freebase-python.googlecode.com/svn/trunk@105 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2 --- .../freebase-images-appengine/freebase/__init__.py | 43 ++ .../freebase/__init__.pyc | Bin 0 -> 610 bytes .../freebase/api/__init__.py | 4 + .../freebase/api/__init__.pyc | Bin 0 -> 327 bytes .../freebase/api/cookie_handlers.py | 215 ++++++ .../freebase/api/cookie_handlers.pyc | Bin 0 -> 7940 bytes .../freebase/api/httpclients.py | 91 +++ .../freebase/api/httpclients.pyc | Bin 0 -> 3872 bytes .../freebase/api/mqlkey.py | 135 ++++ .../freebase/api/mqlkey.pyc | Bin 0 -> 3020 bytes .../freebase/api/session.py | 794 +++++++++++++++++++++ .../freebase/api/session.pyc | Bin 0 -> 23549 bytes .../freebase/fcl/README.txt | 8 + .../freebase/fcl/__init__.py | 0 .../freebase/fcl/commands.py | 688 ++++++++++++++++++ .../freebase/fcl/fbutil.py | 113 +++ .../freebase-images-appengine/freebase/fcl/fcl.py | 276 +++++++ .../freebase/fcl/inspect.py | 218 ++++++ .../freebase/fcl/mktype.py | 179 +++++ .../freebase-images-appengine/freebase/rison.py | 308 ++++++++ .../freebase-images-appengine/freebase/sandbox.py | 68 ++ .../freebase-images-appengine/freebase/sandbox.pyc | Bin 0 -> 583 bytes .../freebase/uritemplate.py | 143 ++++ examples/freebase-images-appengine/images.html | 3 - 24 files changed, 3283 insertions(+), 3 deletions(-) create mode 100755 examples/freebase-images-appengine/freebase/__init__.py create mode 100644 examples/freebase-images-appengine/freebase/__init__.pyc create mode 100755 examples/freebase-images-appengine/freebase/api/__init__.py create mode 100644 examples/freebase-images-appengine/freebase/api/__init__.pyc create mode 100755 examples/freebase-images-appengine/freebase/api/cookie_handlers.py create mode 100644 examples/freebase-images-appengine/freebase/api/cookie_handlers.pyc create mode 100644 examples/freebase-images-appengine/freebase/api/httpclients.py create mode 100644 examples/freebase-images-appengine/freebase/api/httpclients.pyc create mode 100755 examples/freebase-images-appengine/freebase/api/mqlkey.py create mode 100644 examples/freebase-images-appengine/freebase/api/mqlkey.pyc create mode 100644 examples/freebase-images-appengine/freebase/api/session.py create mode 100644 examples/freebase-images-appengine/freebase/api/session.pyc create mode 100644 examples/freebase-images-appengine/freebase/fcl/README.txt create mode 100755 examples/freebase-images-appengine/freebase/fcl/__init__.py create mode 100755 examples/freebase-images-appengine/freebase/fcl/commands.py create mode 100755 examples/freebase-images-appengine/freebase/fcl/fbutil.py create mode 100755 examples/freebase-images-appengine/freebase/fcl/fcl.py create mode 100755 examples/freebase-images-appengine/freebase/fcl/inspect.py create mode 100755 examples/freebase-images-appengine/freebase/fcl/mktype.py create mode 100755 examples/freebase-images-appengine/freebase/rison.py create mode 100644 examples/freebase-images-appengine/freebase/sandbox.py create mode 100644 examples/freebase-images-appengine/freebase/sandbox.pyc create mode 100755 examples/freebase-images-appengine/freebase/uritemplate.py delete mode 100644 examples/freebase-images-appengine/images.html (limited to 'examples/freebase-images-appengine') diff --git a/examples/freebase-images-appengine/freebase/__init__.py b/examples/freebase-images-appengine/freebase/__init__.py new file mode 100755 index 0000000..c55b700 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/__init__.py @@ -0,0 +1,43 @@ +import sys + +from freebase.api.session import HTTPMetawebSession +import sandbox + +__all__ = ["HTTPMetawebSession", "sandbox"] + +_base = HTTPMetawebSession("freebase.com") + +# we want to add base's functions to __init__.py +# so that we can say freebase.func() and really +# just call base.func() + +# a little trick to refer to __init__ +# self isn't defined because __init__ is in +# a world in and of itself +self = sys.modules[__name__] + +for funcname in dir(_base): + + # we only want the 'real' functions + if not funcname.startswith("_"): + func = getattr(_base, funcname) + + # let's make sure we're getting functions + # instead of constants or whatever + if callable(func): + + # we're setting these functions + # so that they can be called like + # freebase.funcname -> base.func() + setattr(self, funcname, func) + + # make sure we import the base's + # functions if we import freebase + __all__.append(funcname) + +# we don't want any self-referencing +# business going. Plus, this is cleaner. +del self + +# we want dir(freebase) to be clean +del funcname, func diff --git a/examples/freebase-images-appengine/freebase/__init__.pyc b/examples/freebase-images-appengine/freebase/__init__.pyc new file mode 100644 index 0000000..5fcaf13 Binary files /dev/null and b/examples/freebase-images-appengine/freebase/__init__.pyc differ diff --git a/examples/freebase-images-appengine/freebase/api/__init__.py b/examples/freebase-images-appengine/freebase/api/__init__.py new file mode 100755 index 0000000..c3adebc --- /dev/null +++ b/examples/freebase-images-appengine/freebase/api/__init__.py @@ -0,0 +1,4 @@ + +from session import HTTPMetawebSession, MetawebError, attrdict + +from mqlkey import quotekey, unquotekey diff --git a/examples/freebase-images-appengine/freebase/api/__init__.pyc b/examples/freebase-images-appengine/freebase/api/__init__.pyc new file mode 100644 index 0000000..e641cba Binary files /dev/null and b/examples/freebase-images-appengine/freebase/api/__init__.pyc differ diff --git a/examples/freebase-images-appengine/freebase/api/cookie_handlers.py b/examples/freebase-images-appengine/freebase/api/cookie_handlers.py new file mode 100755 index 0000000..88f6b4f --- /dev/null +++ b/examples/freebase-images-appengine/freebase/api/cookie_handlers.py @@ -0,0 +1,215 @@ +# ======================================================================== +# Copyright (c) 2007, Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ======================================================================== + +# +# +# httplib2cookie.py allows you to use python's standard +# CookieJar class with httplib2. +# +# + +import re + +try: + from google.appengine.api import urlfetch + Http = object +except ImportError: + pass + +try: + from httplib2 import Http +except ImportError: + pass + +try: + import urllib +except ImportError: + import urllib_stub as urllib +import cookielib + +class DummyRequest(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, url, headers=None): + self.url = url + self.headers = headers + self.origin_req_host = cookielib.request_host(self) + self.type, r = urllib.splittype(url) + self.host, r = urllib.splithost(r) + if self.host: + self.host = urllib.unquote(self.host) + + def get_full_url(self): + return self.url + + def get_origin_req_host(self): + # TODO to match urllib2 this should be different for redirects + return self.origin_req_host + + def get_type(self): + return self.type + + def get_host(self): + return self.host + + def get_header(self, key, default=None): + return self.headers.get(key.lower(), default) + + def has_header(self, key): + return key in self.headers + + def add_unredirected_header(self, key, val): + # TODO this header should not be sent on redirect + self.headers[key.lower()] = val + + def is_unverifiable(self): + # TODO to match urllib2, this should be set to True when the + # request is the result of a redirect + return False + +class DummyHttplib2Response(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def info(self): + return DummyHttplib2Message(self.response) + + +class DummyUrlfetchResponse(object): + """Simulated urllib2.Request object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def info(self): + return DummyUrlfetchMessage(self.response) + + +class DummyHttplib2Message(object): + """Simulated mimetools.Message object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def getheaders(self, k): + k = k.lower() + v = self.response.get(k.lower(), None) + if k not in self.response: + return [] + #return self.response[k].split(re.compile(',\\s*')) + + # httplib2 joins multiple values for the same header + # using ','. but the netscape cookie format uses ',' + # as part of the expires= date format. so we have + # to split carefully here - header.split(',') won't do it. + HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') + return [h[0] for h in HEADERVAL.findall(self.response[k])] + +class DummyUrlfetchMessage(object): + """Simulated mimetools.Message object for httplib2 + + implements only what's necessary for cookielib.CookieJar to work + """ + def __init__(self, response): + self.response = response + + def getheaders(self, k): + k = k.lower() + v = self.response.headers.get(k.lower(), None) + if k not in self.response.headers: + return [] + #return self.response[k].split(re.compile(',\\s*')) + + # httplib2 joins multiple values for the same header + # using ','. but the netscape cookie format uses ',' + # as part of the expires= date format. so we have + # to split carefully here - header.split(',') won't do it. + HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') + return [h[0] for h in HEADERVAL.findall(self.response.headers[k])] + +class CookiefulHttp(Http): + """Subclass of httplib2.Http that keeps cookie state + + constructor takes an optional cookiejar=cookielib.CookieJar + + currently this does not handle redirects completely correctly: + if the server redirects to a different host the original + cookies will still be sent to that host. + """ + def __init__(self, cookiejar=None, **kws): + # note that httplib2.Http is not a new-style-class + Http.__init__(self, **kws) + if cookiejar is None: + cookiejar = cookielib.CookieJar() + self.cookiejar = cookiejar + + def request(self, uri, **kws): + headers = kws.pop('headers', None) + req = DummyRequest(uri, headers) + self.cookiejar.add_cookie_header(req) + headers = req.headers + + (r, body) = Http.request(self, uri, headers=headers, **kws) + + resp = DummyHttplib2Response(r) + self.cookiejar.extract_cookies(resp, req) + + return (r, body) + +class CookiefulUrlfetch(object): + """Class that keeps cookie state + + constructor takes an optional cookiejar=cookielib.CookieJar + """ + # TODO refactor CookefulHttp so that CookiefulUrlfetch can be a subclass of it + def __init__(self, cookiejar=None, **kws): + if cookiejar is None: + cookejar = cookielib.CookieJar() + self.cookejar = cookiejar + + def request(self, uri, **kws): + headers = kws.pop('headers', None) + req = DummyRequest(uri, headers) + self.cookejar.add_cookie_header(req) + headers = req.headers + + r = urlfetch.fetch(uri, headers=headers, **kws) + + self.cookejar.extract_cookies(DummyUrlfetchResponse(r), req) + return r + diff --git a/examples/freebase-images-appengine/freebase/api/cookie_handlers.pyc b/examples/freebase-images-appengine/freebase/api/cookie_handlers.pyc new file mode 100644 index 0000000..f5ac0e6 Binary files /dev/null and b/examples/freebase-images-appengine/freebase/api/cookie_handlers.pyc differ diff --git a/examples/freebase-images-appengine/freebase/api/httpclients.py b/examples/freebase-images-appengine/freebase/api/httpclients.py new file mode 100644 index 0000000..48af9b6 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/api/httpclients.py @@ -0,0 +1,91 @@ +try: + from google.appengine.api import urlfetch + from cookie_handlers import CookiefulUrlfetch +except: + pass + +try: + import httplib2 + from cookie_handlers import CookiefulHttp +except: + pass + +try: + import urllib2 + import socket +except: + pass + +import logging +import re + +class Urllib2Client(object): + def __init__(self, cookiejar, rse): + cookiespy = urllib2.HTTPCookieProcessor(cookiejar) + self.opener = urllib2.build_opener(cookiespy) + self._raise_service_error = rse + self.log = logging.getLogger() + + def __call__(self, url, method, body, headers): + req = urllib2.Request(url, body, headers) + + try: + resp = self.opener.open(req) + + except socket.error, e: + self.log.error('SOCKET FAILURE: %s', e.fp.read()) + raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) + + except urllib2.HTTPError, e: + self.log.error('HTTP ERROR: %s', e) + self._raise_service_error(url, e.code, e.info().type, e.fp.read()) + + for header in resp.info().headers: + self.log.debug('HTTP HEADER %s', header) + name, value = re.split("[:\n\r]", header, 1) + if name.lower() == 'x-metaweb-tid': + self.tid = value.strip() + + return (resp, resp.read()) + +class Httplib2Client(object): + def __init__(self, cookiejar, rse): + self.cookiejar = cookiejar + self._raise_service_error = rse + self.httpclient = CookiefulHttp(cookiejar=self.cookiejar) + + def __call__(self, url, method, body, headers): + try: + resp, content = self.httpclient.request(url, method=method, + body=body, headers=headers) + if (resp.status != 200): + self._raise_service_error(url, resp.status, resp['content-type'], content) + + except socket.error, e: + self.log.error('SOCKET FAILURE: %s', e.fp.read()) + raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) + + except httplib2.HttpLib2ErrorWithResponse, e: + self._raise_service_error(url, resp.status, resp['content-type'], content) + except httplib2.HttpLib2Error, e: + raise MetawebError(u'HTTP error: %s' % (e,)) + + #tid = resp.get('x-metaweb-tid', None) + + return (resp, content) + + +class UrlfetchClient(object): + def __init__(self, cookiejar, rse): + self.cookiejar = cookiejar + self._raise_service_error = rse + self.httpclient = CookiefulUrlfetch(cookiejar=self.cookiejar) + + def __call__(self, url, method, body, headers): + resp = self.httpclient.request(url, payload=body, method=method, headers=headers) + + if resp.status_code != 200: + self._raise_service_error(url, resp.status_code, resp.headers['content-type'], resp.body) + + return (resp, resp.content) + diff --git a/examples/freebase-images-appengine/freebase/api/httpclients.pyc b/examples/freebase-images-appengine/freebase/api/httpclients.pyc new file mode 100644 index 0000000..d28177b Binary files /dev/null and b/examples/freebase-images-appengine/freebase/api/httpclients.pyc differ diff --git a/examples/freebase-images-appengine/freebase/api/mqlkey.py b/examples/freebase-images-appengine/freebase/api/mqlkey.py new file mode 100755 index 0000000..d7dd01b --- /dev/null +++ b/examples/freebase-images-appengine/freebase/api/mqlkey.py @@ -0,0 +1,135 @@ +# ======================================================================== +# Copyright (c) 2007, Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ======================================================================== + +import string +import re + +def quotekey(ustr): + """ + quote a unicode string to turn it into a valid namespace key + + """ + valid_always = string.ascii_letters + string.digits + valid_interior_only = valid_always + '_-' + + if isinstance(ustr, str): + s = unicode(ustr,'utf-8') + elif isinstance(ustr, unicode): + s = ustr + else: + raise ValueError, 'quotekey() expects utf-8 string or unicode' + + output = [] + if s[0] in valid_always: + output.append(s[0]) + else: + output.append('$%04X' % ord(s[0])) + + for c in s[1:-1]: + if c in valid_interior_only: + output.append(c) + else: + output.append('$%04X' % ord(c)) + + if len(s) > 1: + if s[-1] in valid_always: + output.append(s[-1]) + else: + output.append('$%04X' % ord(s[-1])) + + return str(''.join(output)) + + +def unquotekey(key, encoding=None): + """ + unquote a namespace key and turn it into a unicode string + """ + + valid_always = string.ascii_letters + string.digits + + output = [] + i = 0 + while i < len(key): + if key[i] in valid_always: + output.append(key[i]) + i += 1 + elif key[i] in '_-' and i != 0 and i != len(key): + output.append(key[i]) + i += 1 + elif key[i] == '$' and i+4 < len(key): + # may raise ValueError if there are invalid characters + output.append(unichr(int(key[i+1:i+5],16))) + i += 5 + else: + raise ValueError, "unquote key saw invalid character '%s' at position %d" % (key[i], i) + + ustr = u''.join(output) + + if encoding is None: + return ustr + + return ustr.encode(encoding) + + +# should this also include "'()" into safe? +def urlencode_pathseg(data): + ''' + urlencode for placement between slashes in an url. + ''' + if isinstance(data, unicode): + data = data.encode('utf_8') + return urllib.quote(data, "~:@$!*,;=&+") + + +def id_to_urlid(id): + """ + convert a mql id to an id suitable for embedding in a url path. + """ + + segs = id.split('/') + + assert isinstance(id, str) and id != '', 'bad id "%s"' % id + + if id[0] == '~': + assert len(segs) == 1 + # assume valid, should check + return id + + if id[0] == '#': + assert len(segs) == 1 + # assume valid, should check + return '%23' + id[1:] + + if id[0] != '/': + raise ValueError, 'unknown id format %s' % id + + # ok, we have a slash-path + # requote components as keys and rejoin. + # urlids do not have leading slashes!!! + return '/'.join(urlencode_pathseg(unquotekey(seg)) for seg in segs[1:]) + diff --git a/examples/freebase-images-appengine/freebase/api/mqlkey.pyc b/examples/freebase-images-appengine/freebase/api/mqlkey.pyc new file mode 100644 index 0000000..66cdde2 Binary files /dev/null and b/examples/freebase-images-appengine/freebase/api/mqlkey.pyc differ diff --git a/examples/freebase-images-appengine/freebase/api/session.py b/examples/freebase-images-appengine/freebase/api/session.py new file mode 100644 index 0000000..be3d997 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/api/session.py @@ -0,0 +1,794 @@ +# ================================================================== +# Copyright (c) 2007, Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + +""" +declarations for external metaweb api. + + + from metaweb.api import HTTPMetawebSession + + mss = HTTPMetawebSession('sandbox.freebase.com') + print mss.mqlread([dict(name=None, type='/type/type')]) +""" + + + +__all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict'] +__version__ = '0.1' + +import os, sys, re +import cookielib +try: + import simplejson +except ImportError: + try: + # appengine provides simplejson at django.utils.simplejson + from django.utils import simplejson + except ImportError: + raise Exception("unable to import simplejson") +try: + from urllib import quote as urlquote +except ImportError: + from urlib_stub import quote as urlquote +import pprint +import socket +import logging + +class Delayed(object): + """ + Wrapper for callables in log statements. Avoids actually making + the call until the result is turned into a string. + + A few examples: + + simplejson.dumps is never called because the logger never + tries to format the result + >>> logging.debug(Delayed(simplejson.dumps, q)) + + This time simplejson.dumps() is actually called: + >>> logging.warn(Delayed(simplejson.dumps, q)) + + """ + def __init__(self, f, *args, **kwds): + self.f = f + self.args = args + self.kwds = kwds + + def __str__(self): + return str(self.f(*self.args, **self.kwds)) + +def logformat(result): + """ + Format the dict/list as a json object + """ + rstr = simplejson.dumps(result, indent=2) + if rstr[0] == '{': + rstr = rstr[1:-2] + return rstr + +from httpclients import Httplib2Client, Urllib2Client, UrlfetchClient + +# Check for urlfetch first so that urlfetch is used when running the appengine SDK +try: + import google.appengine.api.urlfetch + from cookie_handlers import CookiefulUrlfetch + http_client = UrlfetchClient +except ImportError: + try: + import httplib2 + from cookie_handlers import CookiefulHttp + http_client = Httplib2Client + except ImportError: + import urllib2 + httplib2 = None + CookiefulHttp = None + http_client = Urllib2Client + +# remove whitespace from json encoded output +simplejson.JSONEncoder.item_separator = ',' +simplejson.JSONEncoder.key_separator = ':' +# don't escape slashes, we're not pasting into script tags here. +if simplejson.encoder.ESCAPE_DCT.get('/', None) == r'\/': + simplejson.encoder.ESCAPE_DCT['/'] = '/' + +def urlencode_weak(s): + return urlquote(s, safe=',/:$') + + +# from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/361668 +class attrdict(dict): + """A dict whose items can also be accessed as member variables. + + >>> d = attrdict(a=1, b=2) + >>> d['c'] = 3 + >>> print d.a, d.b, d.c + 1 2 3 + >>> d.b = 10 + >>> print d['b'] + 10 + + # but be careful, it's easy to hide methods + >>> print d.get('c') + 3 + >>> d['get'] = 4 + >>> print d.get('a') + Traceback (most recent call last): + TypeError: 'int' object is not callable + """ + def __init__(self, *args, **kwargs): + dict.__init__(self, *args, **kwargs) + self.__dict__ = self + + + +# TODO expose the common parts of the result envelope +class MetawebError(Exception): + """ + an error report from the metaweb service. + """ + pass + + + +# TODO right now this is a completely unnecessary superclass. +# is there enough common behavior between session types +# to justify it? +class MetawebSession(object): + """ + MetawebSession is the base class for MetawebSession, subclassed for + different connection types. Only http is available externally. + + This is more of an interface than a class + """ + + # interface definition here... + + +# from httplib2 +NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') +def _normalize_headers(headers): + return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) + +class HTTPMetawebSession(MetawebSession): + """ + a MetawebSession is a request/response queue. + + this version uses the HTTP api, and is synchronous. + """ + # share cookies across sessions, so that different sessions can + # see each other's writes immediately. + _default_cookiejar = cookielib.CookieJar() + + def __init__(self, service_url, username=None, password=None, prev_session=None, cookiejar=None, cookiefile=None): + """ + create a new MetawebSession for interacting with the Metaweb. + + a new session will inherit state from prev_session if present, + """ + super(HTTPMetawebSession, self).__init__() + + self.log = logging.getLogger() + + assert not service_url.endswith('/') + if not '/' in service_url: # plain host:port + service_url = 'http://' + service_url + + self.service_url = service_url + + self.username = username + self.password = password + + self.tid = None + + if prev_session: + self.service_url = prev.service_url + + if cookiefile is not None: + cookiejar = self.open_cookie_file(cookiefile) + + if cookiejar is not None: + self.cookiejar = cookiejar + elif prev_session: + self.cookiejar = prev_session.cookiejar + else: + self.cookiejar = self._default_cookiejar + + self._http_request = http_client(self.cookiejar, self._raise_service_error) + + + def open_cookie_file(self, cookiefile=None): + if cookiefile is None or cookiefile == '': + if os.environ.has_key('HOME'): + cookiefile = os.path.join(os.environ['HOME'], '.pyfreebase/cookiejar') + else: + raise MetawebError("no cookiefile specified and no $HOME/.pyfreebase directory" % cookiefile) + + cookiejar = cookielib.LWPCookieJar(cookiefile) + if os.path.exists(cookiefile): + cookiejar.load(ignore_discard=True) + + return cookiejar + + + def _httpreq(self, service_path, method='GET', body=None, form=None, + headers=None): + """ + make an http request to the service. + + form arguments are encoded in the url, even for POST, if a non-form + content-type is given for the body. + + returns a pair (resp, body) + + resp is the response object and may be different depending + on whether urllib2 or httplib2 is in use? + """ + + if method == 'GET': + assert body is None + if method != "GET" and method != "POST": + assert 0, 'unknown method %s' % method + + url = self.service_url + service_path + + if headers is None: + headers = {} + else: + headers = _normalize_headers(headers) + + # this is a lousy way to parse Content-Type, where is the library? + ct = headers.get('content-type', None) + if ct is not None: + ct = ct.split(';')[0] + + if body is not None: + # if body is provided, content-type had better be too + assert ct is not None + + if form is not None: + qstr = '&'.join(['%s=%s' % (urlencode_weak(unicode(k)), urlencode_weak(unicode(v))) + for k,v in form.items()]) + if method == 'POST': + # put the args on the url if we're putting something else + # in the body. this is used to add args to raw uploads. + if body is not None: + url += '?' + qstr + else: + if ct is None: + ct = 'application/x-www-form-urlencoded' + headers['content-type'] = ct + '; charset=utf-8' + + if ct == 'multipart/form-encoded': + # TODO handle this case + raise NotImplementedError + elif ct == 'application/x-www-form-urlencoded': + body = qstr + else: + # for all methods other than POST, use the url + url += '?' + qstr + + + # assure the service that this isn't a CSRF form submission + headers['x-metaweb-request'] = 'Python' + + if 'user-agent' not in headers: + headers['user-agent'] = 'python freebase.api-%s' % __version__ + + #if self.tid is not None: + # headers['x-metaweb-tid'] = self.tid + + ####### DEBUG MESSAGE - should check log level before generating + if form is None: + formstr = '' + else: + formstr = '\nFORM:\n ' + '\n '.join(['%s=%s' % (k,v) + for k,v in form.items()]) + if headers is None: + headerstr = '' + else: + headerstr = '\nHEADERS:\n ' + '\n '.join([('%s: %s' % (k,v)) + for k,v in headers.items()]) + self.log.info('%s %s%s%s', method, url, formstr, headerstr) + ####### + + return self._http_request(url, method, body, headers) + + def _raise_service_error(self, url, status, ctype, body): + + is_jsbody = (ctype.endswith('javascript') + or ctype.endswith('json')) + if str(status) == '400' and is_jsbody: + r = self._loadjson(body) + msg = r.messages[0] + raise MetawebError(u'%s %s %r' % (msg.get('code',''), msg.message, msg.info)) + + raise MetawebError, 'request failed: %s: %r %r' % (url, status, body) + + def _httpreq_json(self, *args, **kws): + resp, body = self._httpreq(*args, **kws) + return self._loadjson(body) + + def _loadjson(self, json): + # TODO really this should be accomplished by hooking + # simplejson to create attrdicts instead of dicts. + def struct2attrdict(st): + """ + copy a json structure, turning all dicts into attrdicts. + + copying descends instances of dict and list, including subclasses. + """ + if isinstance(st, dict): + return attrdict([(k,struct2attrdict(v)) for k,v in st.items()]) + if isinstance(st, list): + return [struct2attrdict(li) for li in st] + return st + + if json == '': + self.log.error('the empty string is not valid json') + raise MetawebError('the empty string is not valid json') + + try: + r = simplejson.loads(json) + except ValueError, e: + self.log.error('error parsing json string %r' % json) + raise MetawebError, 'error parsing JSON string: %s' % e + + return struct2attrdict(r) + + def _check_mqlerror(self, r): + if r.code != '/api/status/ok': + for msg in r.messages: + self.log.error('mql error: %s %s %r' % (msg.code, msg.message, msg.get('query', None))) + raise MetawebError, 'query failed: %s\n%r' % (r.messages[0].code, r.messages[0].get('query', None)) + + def _mqlresult(self, r): + self._check_mqlerror(r) + + self.log.info('result: %s', Delayed(logformat, r)) + + return r.result + + + + def login(self, username=None, password=None): + """sign in to the service. For a more complete description, + see http://www.freebase.com/view/en/api_account_login""" + + service = '/api/account/login' + + username = username or self.username + password = password or self.password + + assert username is not None + assert password is not None + + self.log.debug('LOGIN USERNAME: %s', username) + + r = self._httpreq_json(service, 'POST', + form=dict(username=username, + password=password)) + + if r.code != '/api/status/ok': + raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) + + self.log.debug('LOGIN RESP: %r', r) + self.log.debug('LOGIN COOKIES: %s', self.cookiejar) + + def logout(self): + """logout of the service. For a more complete description, + see http://www.freebase.com/view/en/api_account_logout""" + + service = '/api/account/logout' + + self.log.debug("LOGOUT") + + r = self._httpreq_json(service, 'GET') + + if r.code != '/api/status/ok': + raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) #this should never happen + + def user_info(self, mql_output=None): + """ get user_info. For a more complete description, + see http://www.freebase.com/view/en/api_service_user_info""" + + service = "/api/service/user_info" + + qstr = simplejson.dumps(mql_output) + + r = self._httpreq_json(service, 'POST', form=dict(mql_output=qstr)) + return r + + def loggedin(self): + """check to see whether a user is logged in or not. For a + more complete description, see http://www.freebase.com/view/en/api_account_loggedin""" + + service = "/api/account/loggedin" + try: + r = self._httpreq_json(service, 'GET') + if r.code == "/api/status/ok": + return True + + except MetawebError, me: + return False + + def mqlreaditer(self, sq, asof=None): + """read a structure query.""" + + cursor = True + + while 1: + subq = dict(query=[sq], cursor=cursor, escape=False) + if asof: + subq['as_of_time'] = asof + + qstr = simplejson.dumps(subq) + + service = '/api/service/mqlread' + + r = self._httpreq_json(service, form=dict(query=qstr)) + + for item in self._mqlresult(r): + yield item + + if r['cursor']: + cursor = r['cursor'] + self.log.info('CONTINUING with %s', cursor) + else: + return + + def mqlread(self, sq, asof=None): + """read a structure query. For a more complete description, + see http://www.freebase.com/view/en/api_service_mqlread""" + subq = dict(query=sq, escape=False) + if asof: + subq['as_of_time'] = asof + + if isinstance(sq, list): + subq['cursor'] = True + + service = '/api/service/mqlread' + + self.log.info('%s: %s', + service, + Delayed(logformat, sq)) + + qstr = simplejson.dumps(subq) + r = self._httpreq_json(service, form=dict(query=qstr)) + + return self._mqlresult(r) + + def mqlreadmulti(self, queries, asof=None): + """read a structure query""" + keys = [('q%d' % i) for i,v in enumerate(queries)]; + envelope = {} + for i,sq in enumerate(queries): + subq = dict(query=sq, escape=False) + if asof: + subq['as_of_time'] = asof + + # XXX put this back once mqlreadmulti is working in general + #if isinstance(sq, list): + # subq['cursor'] = True + envelope[keys[i]] = subq + + service = '/api/service/mqlread' + + self.log.info('%s: %s', + service, + Delayed(logformat, envelope)) + + qstr = simplejson.dumps(envelope) + rs = self._httpreq_json(service, form=dict(queries=qstr)) + + self.log.info('%s result: %s', + service, + Delayed(simplejson.dumps, rs, indent=2)) + + return [self._mqlresult(rs[key]) for key in keys] + + def raw(self, id): + """translate blob from id. For a more complete description, + see http://www.freebase.com/view/en/api_trans_raw""" + url = '/api/trans/raw' + urlquote(id) + + self.log.info(url) + + resp, body = self._httpreq(url) + + self.log.info('raw is %d bytes' % len(body)) + + return body + + def blurb(self, id, break_paragraphs=False, maxlength=200): + """translate only the text in blob from id. For a more + complete description, see http://www.freebase.com/view/en/api_trans_blurb""" + url = '/api/trans/blurb' + urlquote(id) + + self.log.info(url) + + resp, body = self._httpreq(url, form=dict(break_paragraphs=break_paragraphs, maxlength=maxlength)) + + self.log.info('blurb is %d bytes' % len(body)) + + return body + + def image_thumb(self, id, maxwidth=None, maxheight=None, mode="fit", onfail=None): + """ given the id of an image, this will return a URL of a thumbnail of the image. + The full details of how the image is cropped and finessed is detailed at + http://www.freebase.com/view/en/api_trans_image_thumb """ + + service = "/api/trans/image_thumb" + assert mode in ["fit", "fill", "fillcrop", "fillcropmid"] + + form = dict(mode=mode) + if maxwidth is not None: + form["maxwidth"] = maxwidth + if maxheight is not None: + form["maxheight"] = maxheight + if onfail is not None: + form["onfail"] = onfail + + resp, body = self._httpreq(service + urlquote(id), form=form) + self.log.info('image is %d bytes' % len(body)) + + return body + + def mqlwrite(self, sq): + """do a mql write. For a more complete description, + see http://www.freebase.com/view/en/api_service_mqlwrite""" + query = dict(query=sq, escape=False) + qstr = simplejson.dumps(query) + + self.log.debug('MQLWRITE: %s', qstr) + + service = '/api/service/mqlwrite' + + self.log.info('%s: %s', + service, + Delayed(logformat,sq)) + + r = self._httpreq_json(service, 'POST', + form=dict(query=qstr)) + + self.log.debug('MQLWRITE RESP: %r', r) + return self._mqlresult(r) + + def mqlcheck(self, sq): + """ See if a write is valid, and see what would happen, but do not + actually do the write """ + + query = dict(query=sq, escape=False) + qstr = simplejson.dumps(query) + + self.log.debug('MQLCHECK: %s', qstr) + + service = '/api/service/mqlcheck' + + self.log.info('%s: %s', + service, + Delayed(logformat, sq)) + + r = self._httpreq_json(service, 'POST', + form=dict(query=qstr)) + + + self.log.debug('MQLCHECK RESP: %r', r) + + return self._mqlresult(r) + + def mqlflush(self): + """ask the service not to hand us old data""" + self.log.debug('MQLFLUSH') + + service = '/api/service/touch' + r = self._httpreq_json(service) + + self._check_mqlerror(r) + return True + + def touch(self): + """ make sure you are accessing the most recent data. For a more + complete description, see http://www.freebase.com/view/en/api_service_touch""" + return self.mqlflush() + + + def upload(self, body, content_type, document_id=False, permission_of=False): + """upload to the metaweb. For a more complete description, + see http://www.freebase.com/view/en/api_service_upload""" + + service = '/api/service/upload' + + self.log.info('POST %s: %s (%d bytes)', + service, content_type, len(body)) + + + headers = {} + if content_type is not None: + headers['content-type'] = content_type + + form = None + if document_id is not False: + if document_id is None: + form = { 'document': '' } + else: + form = { 'document': document_id } + if permission_of is not False: + if form: + form['permission_of'] = permission_of + else: + form = { 'permission_of' : permission_of } + + # note the use of both body and form. + # form parameters get encoded into the URL in this case + r = self._httpreq_json(service, 'POST', + headers=headers, body=body, form=form) + return self._mqlresult(r) + + + def uri_submit(self, URI, document=None, content_type=None): + """ submit a URI to freebase. For a more complete description, + see http://www.freebase.com/edit/topic/en/api_service_uri_submit """ + + service = "/api/service/uri_submit" + + form = dict(uri=URI) + if document is not None: + form["document"] = document + if content_type is not None: + form["content_type"] = content_type + + r = self._httpreq_json(service, 'POST', form=form) + return self._mqlresult(r) + + + def search(self, query, format=None, prefixed=None, limit=20, start=0, + type=None, type_strict="any", domain=None, domain_strict=None, + escape="html", timeout=None, mql_filter=None, mql_output=None): + """ search freebase.com. For a more complete description, + see http://www.freebase.com/view/en/api_service_search""" + + service = "/api/service/search" + + form = dict(query=query) + + if format: + form["format"] = format + if prefixed: + form["prefixed"] = prefixed + if limit: + form["limit"] = limit + if start: + form["start"] = start + if type: + form["type"] = type + if type_strict: + form["type_strict"] = type_strict + if domain: + form["domain"] = domain + if domain_strict: + form["domain_strict"] = domain_strict + if escape: + form["escape"] = escape + if timeout: + form["timeout"] = timeout + if mql_filter: + form["mql_filter"] = mql_filter + if mql_output: + form["mql_output"] = mql_output + + + r = self._httpreq_json(service, 'POST', form=form) + + return self._mqlresult(r) + + + def geosearch(self, location=None, location_type=None, mql_input=None, limit=20, + start=0, type=None, geometry_type=None, intersect=None, mql_filter=None, + within=None, inside=None, order_by=None, count=None, format="json", mql_output=None): + """ perform a geosearch. For a more complete description, + see http://www.freebase.com/api/service/geosearch?help """ + + service = "/api/service/geosearch" + + if location is None and location_type is None and mql_input is None: + raise Exception("You have to give it something to work with") + + form = dict() + + if location: + form["location"] = location + if location_type: + form["location_type"] = location_type + if mql_input: + form["mql_input"] = mql_input + if limit: + form["limit"] = limit + if start: + form["start"] = start + if type: + form["type"] = type + if geometry_type: + form["geometry_type"] = geometry_type + if intersect: + form["intersect"] = intersect + if mql_filter: + form["mql_filter"] = mql_filter + if within: + form["within"] = within + if inside: + form["inside"] = inside + if order_by: + form["order_by"] = order_by + if count: + form["count"] = count + if format: + form["format"] = format + if mql_output: + form["mql_output"] = mql_output + + if format == "json": + r = self._httpreq_json(service, 'POST', form=form) + else: + r = self._httpreq(service, 'POST', form=form) + + return r + + def version(self): + """ get versions for various parts of freebase. For a more + complete description, see http://www.freebase.com/view/en/api_version""" + + service = "/api/version" + r = self._httpreq_json(service) + + return r + + ### DEPRECATED IN API + def reconcile(self, name, etype=['/common/topic']): + """DEPRECATED: reconcile name to guid. For a more complete description, + see http://www.freebase.com/view/en/dataserver_reconciliation""" + + service = '/dataserver/reconciliation' + r = self._httpreq_json(service, 'GET', form={'name':name, 'types':','.join(etype)}) + + + # TODO non-conforming service, fix later + #self._mqlresult(r) + return r + + +if __name__ == '__main__': + console = logging.StreamHandler() + console.setLevel(logging.DEBUG) + + mss = HTTPMetawebSession('sandbox.freebase.com') + + self.mss.log.setLevel(logging.DEBUG) + self.mss.log.addHandler(console) + + + print mss.mqlread([dict(name=None, type='/type/type')]) diff --git a/examples/freebase-images-appengine/freebase/api/session.pyc b/examples/freebase-images-appengine/freebase/api/session.pyc new file mode 100644 index 0000000..9bb0d3f Binary files /dev/null and b/examples/freebase-images-appengine/freebase/api/session.pyc differ diff --git a/examples/freebase-images-appengine/freebase/fcl/README.txt b/examples/freebase-images-appengine/freebase/fcl/README.txt new file mode 100644 index 0000000..2e03d71 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/fcl/README.txt @@ -0,0 +1,8 @@ + +This is a command-line tool for performing various freebase commands. + +It should be installed by default as "fcl". + +Try "fcl help" after installing the freebase-api library. + + diff --git a/examples/freebase-images-appengine/freebase/fcl/__init__.py b/examples/freebase-images-appengine/freebase/fcl/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/examples/freebase-images-appengine/freebase/fcl/commands.py b/examples/freebase-images-appengine/freebase/fcl/commands.py new file mode 100755 index 0000000..0fdc283 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/fcl/commands.py @@ -0,0 +1,688 @@ +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + +import os, sys, re, time +from fbutil import * + +import simplejson + +import freebase.rison as rison +from freebase.api import attrdict + + +def cmd_help(fb, command=None): + """get help on commands + + %prog help [cmd] + """ + + if command is not None: + cmd = fb.commands[command] + print ' %s: %s' % (cmd.name, cmd.shortdoc) + print cmd.doc + return + + print """ +the interface to this tool is loosely modeled on the +"svn" command-line tool. many commands work slightly +differently from their unix and svn equivalents - +read the doc for the command first. + +use "%s help " for help on a particular +subcommand. +""" % fb.progpath + + fb.oparser.print_help() + + print 'available subcommands:' + cmds = sorted(fb.commands.keys()) + + for fk in cmds: + cmd = fb.commands[fk] + print ' %s: %s' % (cmd.name, cmd.shortdoc) + + +def cmd_wikihelp(fb): + """get help on commands in mediawiki markup format + + %prog wikihelp + """ + + print """usage: %s subcommand [ args ... ] + + the interface to this tool is loosely modeled on the + "svn" command-line tool. many commands work slightly + differently from their unix and svn equivalents - + read the doc for the command first. + """ % fb.progpath + + print 'subcommands:' + for fk in sorted(fb.commands.keys()): + cmd = fb.commands[fk] + + # mediawiki markup cares about the difference between + # a blank line and a blank line with a bunch of + # spaces at the start of it. ewww. + doc = re.sub(r'\r?\n {0,8}', '\n ', cmd.doc) + + print '==== %s %s: %s ====' % (fb.progpath, cmd.name, cmd.shortdoc) + print doc + print + + +# this command is disabled because it relies on some svn glue that +# was not sufficiently well thought out. +def old_cmd_pwid(fb): + """show the "current working namespace id" + + %prog pwid + + by default, relative ids can't be resolved. however, if you + run the fb tool from a directory that has the svn property + 'freebase:id' set, relative ids will be resolved relative + to that id. the idea is that you can create an svn tree + that parallels the freebase namespace tree. + + for example: + $ %prog pwid + + $ svn propset freebase:id "/freebase" . + + $ %prog pwid + /freebase + + """ + if fb.cwid is not None: + print fb.cwid + else: + print '' + + +def cmd_ls(fb, path=None): + """list the keys in a namespace + + %prog ls [id] + """ + path = fb.absid(path) + q = {'id': path, + '/type/namespace/keys': [{'value': None, + 'namespace': { + 'id': None, + 'type': [] + }, + 'optional':True + }] + } + + r = fb.mss.mqlread(q) + if r is None: + raise CmdException('query for id %r failed' % path) + + #sys.stdout.write(' '.join([mk.value for mk in r['/type/namespace/keys']])) + + for mk in r['/type/namespace/keys']: + print mk.value + + if 0: + suffix = '' + if ('/type/namespace' in mk.namespace.type + or '/type/domain' in mk.namespace.type): + suffix = '/' + print mk.value, mk.namespace.id+suffix + + +def cmd_mkdir(fb, path): + """create a new freebase namespace + %prog mkdir id + + create a new instance of /type/namespace at the given + point in id space. if id already exists, it should be + a namespace. + """ + path = fb.absid(path) + dir,file = dirsplit(path) + wq = { 'create': 'unless_exists', + 'key':{ + 'connect': 'insert', + 'namespace': dir, + 'value': file + }, + 'name': path, + 'type': '/type/namespace' + } + + r = fb.mss.mqlwrite(wq) + +def cmd_ln(fb, src, dst): + """create a namespace key + %prog ln srcid dstid + + create a new namespace link at dstid to the object + currently at srcid. + """ + src = fb.absid(src) + dst = fb.absid(dst) + dir,file = dirsplit(dst) + wq = { 'id': src, + 'key':{ + 'connect': 'insert', + 'namespace': dir, + 'value': file + } + } + + r = fb.mss.mqlwrite(wq) + + +def cmd_rm(fb, path): + """unlink a namespace key + %prog rm id + + remove the /type/key that connects the given id to its + parent. id must be a path for this to make any sense. + + note that is like unix 'unlink' rather than 'rm'. + it won't complain if the 'subdirectory' contains data, + since that data will still be accessible to other queries. + it's not like 'rm -rf' either, because it doesn't + disturb anything other than the one directory entry. + """ + path = fb.absid(path) + dir,file = dirsplit(path) + + wq = { 'id': path, + 'key':{ + 'connect': 'delete', + 'namespace': dir, + 'value': file + } + } + + r = fb.mss.mqlwrite(wq) + +def cmd_mv(fb, src, dst): + """rename srcid to dstid. + %prog mv srcid dstid + + equivalent to: + $ fb ln + $ fb rm + """ + cmd_ln(fb, src, dst) + cmd_rm(fb, src) + +def cmd_cat(fb, id, include_headers=False): + """download a document from freebase to stdout + %prog cat id + + equivalent to "%prog get id -". + """ + return cmd_get(fb, id, localfile='-', include_headers=include_headers) + +def cmd_get(fb, id, localfile=None, include_headers=False): + """download a file from freebase + %prog get id [localfile] + + download the document or image with the given id from freebase + into localfile. localfile '-' means stdout. localfile + defaults to a file in the current directory with the same name + as the last key in the path, possibly followed by a metadata + extension like .html or .txt. + """ + id = fb.absid(id) + dir,file = dirsplit_unsafe(id) + + def read_content(id, content_only=False): + c = attrdict(id=id) + cq = { 'id': id, + 'type': [], + '/common/document/content': None, + '/common/document/source_uri': None, + '/type/content/media_type': { 'name':None, + 'optional': True }, + #'/type/content/text_encoding': { 'name':None }, + '/type/content/blob_id':None, + } + cd = fb.mss.mqlread(cq) + if '/type/content' in cd.type: + c.media_type = cd['/type/content/media_type'].name + #c.text_encoding = cd['/type/content/text_encoding'].name + c.sha256 = cd['/type/content/blob_id'] + return c + + if content_only: + raise CmdException('%s is not a content id' % id) + + cid = cd['/common/document/content'] + if cid is not None: + return read_content(cid, content_only=True) + + # in this case we don't have a content object + if cd['/common/document/source_uri'] is not None: + return None + + raise CmdException('%s is not a content or document id' % id) + + + content = read_content(id) + log.debug('COBJ %r' % content) + if content is not None: + fileext = media_type_to_extension.get(content.media_type, None) + else: + fileext = None + + if localfile == '-': + ofp = sys.stdout + else: + if localfile is None: + implicit_outfile = True + localfile = file + elif re.match(r'[/\\]$', localfile): + implicit_outfile = True + localfile = localfile + file + else: + implicit_outfile = False + localfile = os.path.abspath(localfile) + + # add file extension based on content-type: + # should be an option to disable this + if implicit_outfile and fileext is not None: + localfile += '.' + fileext + + # if we didn't explicitly name the output file, + # don't destroy an existing file + localfile_base = localfile + count = 0 + while implicit_outfile and os.path.exists(localfile): + count += 1 + localfile = '%s.%d' % (localfile_base, count) + ofp = open(localfile, 'wb') + + body = fb.mss.trans(id) + + if include_headers: + # XXX show content-type, what else? + pass + + ofp.write(body) + + if localfile != '-': + print ('%s saved (%d bytes)' % (localfile, len(body))) + ofp.close() + + +def cmd_put(fb, localfile, id=None, content_type=None): + """upload a document to freebase -- EXPERIMENTAL + %prog put localfile [id] [content-type] + + upload the document or image in localfile to given freebase + id. if localfile is '-' the data will be read from stdin. + + if id is missing or empty, a new document will be created. + later the id might default to something computed from localfile + and any svn attributes it has. + + output: a single line, the id of the document. + """ + if content_type is None: + ext = re.sub('^.*\.([^/.]+)$', r'\1', localfile) + media_type = extension_to_media_type.get(ext, None) + + if media_type is None: + raise CmdException('could not infer a media type from extension %r: please specify it' + % ext) + + if media_type.startswith('text/'): + # this is a bad assumption. should sniff it? + text_encoding = 'utf-8' + content_type = '%s;charset=%s' % (media_type, text_encoding) + else: + content_type = media_type + + new_id = None + if id is not None: + idinfo = fb.mss.mqlread({ 'id': id, 'type': '/common/document' }) + if idinfo is None: + new_id = id + id = None + + body = open(localfile, 'rb').read() + r = fb.mss.upload(body, content_type, document_id=id) + + if new_id is None: + print r.document + else: + cmd_ln(fb, r.document, new_id) + print new_id + + +def cmd_dump(fb, id): + """show all properties of a freebase object + %prog dump object_id + """ + id = fb.absid(id) + + import inspect + + r = inspect.inspect_object(fb.mss, id) + if r is None: + raise CmdException('no match for id %r' % id) + + for k in sorted(r.keys()): + vs = r[k] + for v in vs: + id = v.get('id', '') + + name = '%r' % (v.get('name') or v.get('value')) + if name == 'None': name = '' + + type = v.get('type', '') + if type == '/type/text': + extra = v.get('lang', '') + elif type == '/type/key': + extra = v.get('namespace', '') + else: + extra = '' + + fb.trow(k, id, name, type, extra) + + +def cmd_pget(fb, id, propid): + """get a property of a freebase object -- EXPERIMENTAL + %prog pget object_id property_id + + get the property named by property_id from the object. + + XXX output quoting is not well specified. + + property_id must be a fully qualified id for now. + + prints one line for each match. + + if propid ends in '*' this does a wildcard for a particular type. + """ + id = fb.absid(id) + proptype, propkey = dirsplit(propid) + + if propkey != '*': + # look up the prop + q = { 'id': id, + propid: [{}], + } + r = fb.mss.mqlread(q) + for v in r[propid]: + if 'value' in v: + print v.value + else: + print v.id + + else: + # look up the prop + q = { 'id': id, + '*': [{}], + } + if isinstance(proptype, basestring): + q['type'] = proptype + + r = fb.mss.mqlread(q) + + for k in sorted(r.keys()): + v = r[k]; + if 'value' in v: + print '%s %s' % (k, v.value) + else: + print '%s %s' % (k, v.id) + +def cmd_pdel(fb, id, propid, oldval): + """delete a property of a freebase object -- EXPERIMENTAL + %prog pdel object_id property_id oldvalue + + set the property named by property_id on the object. + value is an id or a json value. XXX this is ambiguous. + + property_id must be a fully qualified id for now. + + for now you need to provide a "oldval" argument, + later this tool will query and perhaps prompt if the + deletion is ambiguous. + + prints a single line, either 'deleted' or 'missing' + """ + return cmd_pset(fb, id, propid, None, oldval) + + +def cmd_touch(fb): + """bypass any cached query results the service may have. use sparingly. + """ + fb.mss.mqlflush() + + +def cmd_pset(fb, id, propkey, val, oldval=None, extra=None): + """set a property of a freebase object -- EXPERIMENTAL + %prog pset object_id property_id value + + set the property named by property_id on the object. + value is an id or a json value. XXX this is ambiguous. + + property_id must be a fully qualified id for now. + + if the property should be a unique property, this will + write with 'connect:update'. if the property may have + multiple, it is written with 'connect:insert'. + + prints a single line, either 'inserted' or 'present' + """ + id = fb.absid(id) + + propid = fb.absprop(propkey) + + # look up the prop + pq = { 'id': propid, + 'type': '/type/property', + 'name': None, + 'unique': None, + 'expected_type': { + 'id': None, + 'name': None, + 'default_property': None, + 'optional': True, + }, + } + + prop = fb.mss.mqlread(pq) + + if prop is None: + raise CmdException('can\'t resolve property key %r - use an absolute id' % propid); + + if propid.startswith('/type/object/') or propid.startswith('/type/value/'): + propkey = re.sub('/type/[^/]+/', '', propid); + else: + propkey = propid + + wq = { 'id': id, + propkey: { + } + } + + if val is None: + val = oldval + wq[propkey]['connect'] = 'delete' + elif prop.unique: + wq[propkey]['connect'] = 'update' + else: + wq[propkey]['connect'] = 'insert' + + if prop.expected_type is None: + wq[propkey]['id'] = val + elif prop.expected_type.id not in value_types: + wq[propkey]['id'] = val + else: + wq[propkey]['value'] = val + + if prop.expected_type.id == '/type/text': + if extra is not None: + lang = extra + else: + lang = '/lang/en' + wq[propkey]['lang'] = lang + + if prop.expected_type.id == '/type/key': + if extra is not None: + wq[propkey]['namespace'] = extra + else: + raise CmdException('must specify a namespace to pset /type/key') + + r = fb.mss.mqlwrite(wq) + print r[propkey]['connect'] + +def cmd_login(fb, username=None, password=None): + """login to the freebase service + %prog login [username [password]] + + cookies are maintained in a file so + they are available to the next invocation. + prompts for username and password if not given + """ + import getpass + if username is None: + sys.stdout.write('freebase.com username: ') + username = sys.stdin.readline() + if not username: + raise CmdException('usernmae required for login') + username = re.compile('\n$').sub('', username) + if password is None: + password = getpass.getpass('freebase.com password: ') + + fb.mss.username = username + fb.mss.password = password + fb.mss.login() + +def cmd_logout(fb): + """logout from the freebase service + %prog logout + + deletes the login cookies + """ + fb.cookiejar.clear(domain=fb.service_host.split(':')[0]) + + +def cmd_find(fb, qstr): + """print all ids matching a given constraint. + + if the query string starts with "{" it is treated as json. + otherwise it is treated as o-rison. + + %prog find + """ + if qstr.startswith('{'): + q = simplejson.loads(qstr) + else: + q = rison.loads('(' + qstr + ')') + + if 'id' not in q: + q['id'] = None + + results = fb.mss.mqlreaditer(q) + for r in results: + print r.id + +def cmd_q(fb, qstr): + """run a freebase query. + + if the query string starts with "{" it is treated as json. + otherwise it is treated as o-rison. + + dump the result as json. + + %prog q + """ + if qstr.startswith('{'): + q = simplejson.loads(qstr) + else: + q = rison.loads('(' + qstr + ')') + + # results could be streamed with a little more work + results = fb.mss.mqlreaditer(q) + print simplejson.dumps(list(results), indent=2) + +def cmd_open(fb, id): + """open a web browser on the given id. works on OSX only for now. + + %prog open /some/id + """ + os.system("open 'http://www.freebase.com/view%s'" % id) + + +def cmd_log(fb, id): + """log changes pertaining to a given id. + + INCOMPLETE + + %prog log /some/id + """ + + null = None + true = True + false = False + + baseq = { + 'type': '/type/link', + 'source': null, + 'master_property': null, + 'attribution': null, + 'timestamp': null, + 'operation': null, + 'valid': null, + 'sort': '-timestamp' + }; + + queries = [ + { + 'target_value': { '*': null }, + 'target': { 'id': null, 'name': null, 'optional': true }, + }, + { + 'target': { 'id': null, 'name': null }, + }] + + for i,q in list(enumerate(queries)): + q.update(baseq) + queries[i] = [q] + + + valuesfrom,linksfrom = fb.mss.mqlreadmulti(queries) + + for link in linksfrom: + # fb.trow(link.master_property.id, ...) + print simplejson.dumps(link, indent=2) + + for link in valuesfrom: + # fb.trow(link.master_property.id, ...) + print simplejson.dumps(link, indent=2) + + diff --git a/examples/freebase-images-appengine/freebase/fcl/fbutil.py b/examples/freebase-images-appengine/freebase/fcl/fbutil.py new file mode 100755 index 0000000..ed1e511 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/fcl/fbutil.py @@ -0,0 +1,113 @@ +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + + +import re + +import logging +log = logging.getLogger() + + +class FbException(Exception): + pass + +class CmdException(Exception): + pass + +media_types = { + 'html': ['text/html'], + 'txt': ['text/plain'], + + 'xml': ['text/xml', + 'application/xml'], + + 'atom': ['application/atom+xml'], + + 'js': ['text/javascript', + 'application/javascript', + 'application/x-javascript'], + + 'json': ['application/json'], + + 'jpg': ['image/jpeg', + 'image/pjpeg'], + + 'gif': ['image/gif'], + + 'png': ['image/png'], + } + +extension_to_media_type = dict([(k,vs[0]) for k,vs in media_types.items()]) +media_type_to_extension = {} +for k,vs in media_types.items(): + for v in vs: + media_type_to_extension[v] = k + + + + +DIRSPLIT = re.compile(r'^(.+)/([^/]+)$') + +def dirsplit_unsafe(id): + m = DIRSPLIT.match(id) + if m is None: + return (None, id) + dir,file = m.groups() + return (dir,file) + +def dirsplit(id): + dir,file = dirsplit_unsafe(id) + if dir == '/guid': + raise FbException('%r is not a freebase keypath' % (id,)) + return (dir,file) + +value_types = [ + '/type/text', + '/type/key', + '/type/rawstring', + '/type/float', + '/type/int', + '/type/boolean', + '/type/uri', + '/type/datetime', + '/type/id', + '/type/enumeration', + ] + +default_propkeys = { + 'value': '/type/value/value', + 'id': '/type/object/id', + 'guid': '/type/object/guid', + 'type': '/type/object/type', + 'name': '/type/object/name', + 'key': '/type/object/key', + 'timestamp': '/type/object/timestamp', + 'permission': '/type/object/permission', + 'creator': '/type/object/creator', + 'attribution': '/type/object/attribution' +}; diff --git a/examples/freebase-images-appengine/freebase/fcl/fcl.py b/examples/freebase-images-appengine/freebase/fcl/fcl.py new file mode 100755 index 0000000..a0299b8 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/fcl/fcl.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + +import os, sys, re, time, stat + +from optparse import OptionParser +import getpass +import cookielib +import logging +import simplejson + + +from fbutil import FbException, CmdException, log, default_propkeys +console = logging.StreamHandler() +log.addHandler(console) + +from freebase.api import HTTPMetawebSession, MetawebError, attrdict + +_cookiedir = None +if os.environ.has_key('HOME'): + _cookiedir = os.path.join(os.environ['HOME'], '.pyfreebase') + + +class Command(object): + def __init__(self, module, name, func): + self.module = module + self.name = name + self.func = func + + PROG = re.compile(r'\%prog') + NEWLINE = re.compile(r'(?:\r?\n)+') + + # fill in self.doc + if isinstance(func.__doc__, basestring): + doc = PROG.sub('fcl', func.__doc__ + '\n') + self.shortdoc, self.doc = NEWLINE.split(doc, 1) + else: + self.shortdoc = '(missing documentation)' + self.doc = '(missing documentation)' + + + +class FbCommandHandler(object): + + def __init__(self): + self.service_host = 'www.freebase.com' + self.cookiejar = None + self.cwid = '' + self.progpath = 'fcl' + self.commands = {} + + self.cookiefile = None + if _cookiedir is not None: + self.cookiefile = os.path.join(_cookiedir, 'cookiejar') + + + def init(self): + if self.cookiefile is not None: + self.cookiejar = cookielib.LWPCookieJar(self.cookiefile) + if os.path.exists(self.cookiefile): + try: + self.cookiejar.load(ignore_discard=True) + except cookielib.LoadError: + log.warn('error loading cookies') + + #print 'start cookies %r' % self.cookiejar + + self.mss = HTTPMetawebSession(self.service_host, + cookiejar=self.cookiejar) + + + def absid(self, path): + if path is None: + path = '' + if path.startswith('/'): + return path + + if not isinstance(self.cwid, basestring) or not self.cwid.startswith('/'): + # svn cwid support is disabled because it relies on some svn glue that + # was not sufficiently well thought out. + # raise CmdException("can't resolve relative id %r without cwid - see 'fcl help pwid'" % (path)) + raise CmdException("no support for relative id %r" % (path)) + + if path == '' or path == '.': + return self.cwid + + return self.cwid + '/' + path + + + def absprop(self, propkey): + if propkey.startswith('/'): + return propkey + + # check schemas of /type/object and /type/value, + # as well as other reserved names + if propkey in default_propkeys: + return default_propkeys[propkey] + + return self.absid(propkey) + + def thead(self, *args): + strs = ['%r' % arg + for arg in args] + print '\t'.join(strs) + + def trow(self, *args): + print '\t'.join(args) + return + strs = ['%r' % arg + for arg in args] + print '\t'.join(strs) + + def save(self): + #print 'end cookies %r' % self.cookiejar + if _cookiedir and self.cookiefile.startswith(_cookiedir): + # create private cookiedir if needed + if not os.path.exists(_cookiedir): + os.mkdir(_cookiedir, 0700) + os.chmod(_cookiedir, stat.S_IRWXU) + + if self.cookiejar is None: + return + + self.cookiejar.save(ignore_discard=True) + + # save the cwd and other state too + + + def import_commands(self, modname): + """ + import new fb commands from a file + """ + namespace = {} + + pyimport = 'from %s import *' % modname + exec pyimport in namespace + mod = sys.modules.get(modname) + + commands = [Command(mod, k[4:], getattr(mod, k)) + for k in getattr(mod, '__all__', dir(mod)) + if (k.startswith('cmd_') + and callable(getattr(mod, k)))] + + + for cmd in commands: + log.info('importing %r' % ((cmd.name, cmd.func),)) + self.commands[cmd.name] = cmd + + log.info('imported %r from %r' % (modname, mod.__file__)) + + + def dispatch(self, cmd, args): + if cmd in self.commands: + try: + self.commands[cmd].func(self, *args) + except KeyboardInterrupt, e: + sys.stderr.write('%s\n' % (str(e),)) + except FbException, e: + sys.stderr.write('%s\n' % (str(e),)) + except CmdException, e: + sys.stderr.write('%s\n' % (str(e),)) + except MetawebError, e: + sys.stderr.write('%s\n' % (str(e),)) + else: + self.oparser.error('unknown subcommand %r, try "%s help"' % (cmd, self.progpath)) + + self.save() + + + def cmdline_main(self): + op = OptionParser(usage='%prog [options] command [args...] ') + self.oparser = op + + op.disable_interspersed_args() + + op.add_option('-d', '--debug', dest='debug', + default=False, action='store_true', + help='turn on debugging output') + + op.add_option('-v', '--verbose', dest='verbose', + default=False, action='store_true', + help='verbose output') + + op.add_option('-V', '--very-verbose', dest='very_verbose', + default=False, action='store_true', + help='lots of debug output') + + op.add_option('-s', '--service', dest='service_host', + metavar='HOST', + default=self.service_host, + help='Freebase HTTP service address:port') + + op.add_option('-S', '--sandbox', dest='use_sandbox', + default=False, action='store_true', + help='shortcut for --service=sandbox.freebase.com') + + op.add_option('-c', '--cookiejar', dest='cookiefile', + metavar='FILE', + default=self.cookiefile, + help='Cookie storage file (will be created if missing)') + options,args = op.parse_args() + + if len(args) < 1: + op.error('required subcommand missing') + + + loglevel = logging.WARNING + if options.verbose: + loglevel = logging.INFO + if options.very_verbose: + loglevel = logging.DEBUG + + console.setLevel(loglevel) + log.setLevel(loglevel) + + if options.use_sandbox: + self.service_host = 'sandbox.freebase.com' + else: + self.service_host = options.service_host + + self.cookiefile = options.cookiefile + #self.progpath = sys.argv[0] + + self.init() + + self.mss.log.setLevel(loglevel) + self.mss.log.addHandler(console) + + self.import_commands('freebase.fcl.commands') + self.import_commands('freebase.fcl.mktype') + + cmd = args.pop(0) + self.dispatch(cmd, args) + +# entry point for script +def main(): + try: + # turn off crlf output on windows so we work properly + # with unix tools. + import msvcrt + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) + except ImportError: + pass + + fb = FbCommandHandler() + fb.cmdline_main() + +if __name__ == '__main__': + main() diff --git a/examples/freebase-images-appengine/freebase/fcl/inspect.py b/examples/freebase-images-appengine/freebase/fcl/inspect.py new file mode 100755 index 0000000..8cd04d2 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/fcl/inspect.py @@ -0,0 +1,218 @@ +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + + +# +# +# wrap all the nastiness needed for a general mql inspect query +# +# + +import os, sys, re + + +null = None +true = True +false = False + +inspect_query = { + 'name': null, + 'type': [], + + '/type/reflect/any_master': [{ + 'optional':true, + 'id': null, + 'name': null, + 'link': { + 'master_property': { + 'id': null, + 'schema': null + } + } + }], + + '/type/reflect/any_reverse': [{ + 'optional':true, + 'id': null, + 'name': null, + 'link': { + 'master_property': { + 'id':null, + 'schema': null, + 'expected_type': null, + 'reverse_property': { + 'id': null, + 'schema': null, + 'optional': true + } + } + } + }], + + '/type/reflect/any_value': [{ + 'optional':true, + 'value': null, + 'link': { + 'master_property': { + 'id':null, + 'schema': null, + 'expected_type': null + }, + } + }], + + 't:/type/reflect/any_value': [{ + 'optional':true, + 'type': '/type/text', + 'value': null, + 'lang': null, + 'link': { + 'master_property': { + 'id':null, + 'schema': null + }, + } + }], + + '/type/object/creator': [{ + 'optional':true, + 'id':null, + 'name':null + }], + '/type/object/timestamp': [{ + 'optional':true, + 'value': null, + }], + + '/type/object/key': [{ + 'optional':true, + 'value': null, + 'namespace': null + }], + '/type/namespace/keys': [{ + 'optional':true, + 'value': null, + 'namespace': null + }] +} + + +def transform_result(result): + proptypes = {} + props = {} + + # copy a property from a /type/reflect clause + def pushtype(propdesc, prop): + tid = propdesc['schema'] + propid = propdesc['id'] + + if isinstance(prop, dict): + prop = dict(prop) + if 'link' in prop: + prop.pop('link') + + if tid not in proptypes: + proptypes[tid] = {} + if propid not in proptypes[tid]: + proptypes[tid][propid] = [] + + if propid not in props: + props[propid] = [] + props[propid].append(prop) + + # copy a property that isn't enumerated by /type/reflect + def pushprop(propid): + ps = result[propid] + if ps is None: + return + + # hack to infer the schema from id, not always reliable! + schema = re.sub(r'/[^/]+$', '', propid) + keyprop = dict(id=propid, schema=schema) + for p in ps: + pushtype(keyprop, p) + + ps = result['/type/reflect/any_master'] or [] + for p in ps: + propdesc = p.link.master_property + pushtype(propdesc, p) + + # non-text non-key values + ps = result['/type/reflect/any_value'] or [] + for p in ps: + propdesc = p.link.master_property + + # /type/text values are queried specially + # so that we can get the lang, so ignore + # them here. + if propdesc.expected_type == '/type/text': + continue + + pushtype(propdesc, p) + + # text values + ps = result['t:/type/reflect/any_value'] or [] + for p in ps: + propdesc = p.link.master_property + pushtype(propdesc, p) + + pushprop('/type/object/creator') + pushprop('/type/object/timestamp') + pushprop('/type/object/key') + pushprop('/type/namespace/keys') + + # now the reverse properties + ps = result['/type/reflect/any_reverse'] or [] + for prop in ps: + propdesc = prop.link.master_property.reverse_property + + # synthetic property descriptor for the reverse of + # a property with no reverse descriptor. + # note the bogus id starting with '-'. + if propdesc is None: + # schema = prop.link.master_property.expected_type + # if schema is None: + # schema = 'other' + + schema = 'other' + propdesc = dict(id='-' + prop.link.master_property.id, + schema=schema) + + pushtype(propdesc, prop) + + #return proptypes + return props + + +def inspect_object(mss, id): + q = dict(inspect_query) + q['id'] = id + r = mss.mqlread(q) + if r is None: + return None + return transform_result(r) diff --git a/examples/freebase-images-appengine/freebase/fcl/mktype.py b/examples/freebase-images-appengine/freebase/fcl/mktype.py new file mode 100755 index 0000000..30f8df7 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/fcl/mktype.py @@ -0,0 +1,179 @@ +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + +import os, sys, re, time +from fbutil import * + + +def cmd_mkobj(fb, id, typeid='/common/topic', name=''): + """create a new object with a given type -- EXPERIMENTAL + %prog mkobj new_id typeid name + + create a new object with type typeid at the given + namespace location. + + if present, name gives the display name of the new object. + + """ + id = fb.absid(id) + nsid, key = dirsplit(id) + + typeid = fb.absid(typeid) + + if name == '': + name = key + + wq = { 'create': 'unless_exists', + 'id': None, + 'name': name, + 'type': typeid, + 'key':{ + 'namespace': nsid, + 'value': key + }, + } + + # TODO add included types + + r = fb.mss.mqlwrite(wq) + print r.id,r.create + +def cmd_mktype(fb, id, name=''): + """create a new type -- EXPERIMENTAL + %prog mktype new_id name + + create a new object with type Type at the given + namespace location. + + this doesn't create any type hints. + + if present, name gives the display name of the new property + """ + id = fb.absid(id) + + nsid, key = dirsplit(id) + + if name == '': + name = key + + wq = { 'create': 'unless_exists', + 'id': None, + 'name': name, + 'type': '/type/type', + 'key':{ + 'namespace': nsid, + 'value': key + }, + } + + r = fb.mss.mqlwrite(wq) + print r.id,r.create + +def mkprop(fb, typeid, key, name='', vtype=None, master_property=None): + """helper to create a new property + """ + if name == '': + name = key + + wq = { 'create': 'unless_exists', + 'id': None, + 'type': '/type/property', + 'name': name, + 'schema': typeid, + 'key': { + 'namespace': typeid, + 'value': key + } + } + + if vtype is not None: + wq['expected_type'] = vtype + if master_property is not None: + wq['master_property'] = master_property + + return fb.mss.mqlwrite(wq) + + +def cmd_mkprop(fb, id, name='', vtype=None, revkey=None, revname=''): + """create a new property -- EXPERIMENTAL + %prog mkprop new_id [name] [expected_type] [reverse_property] [reverse_name] + + create a new object with type Property at the given + location. creates both the "schema" and "key" links + for the property, but doesn't create any freebase property + hints. + + if present, name gives the display name of the new property + """ + id = fb.absid(id) + if vtype is not None: + vtype = fb.absid(vtype) + + typeid, key = dirsplit(id) + + + r = mkprop(fb, typeid, key, name, vtype) + + # write the reverse property if specified + + print r.id, r.create + + if revkey is None: + return + + assert vtype is not None + + rr = mkprop(fb, vtype, revkey, revname, typeid, id) + print rr.id, rr.create + + +def cmd_publish_type(fb, typeid): + """try to publish a freebase type for the client + %prog publish_type typeid + + set /freebase/type_profile/published to the /freebase/type_status + instance named 'Published' + + + should also try to set the domain to some namespace that + has type:/type/domain + + """ + id = fb.absid(typeid) + + w = { + 'id': id, + '/freebase/type_profile/published': { + 'connect': 'insert', + 'type': '/freebase/type_status', + 'name': 'Published' + } + } + r = fb.mss.mqlwrite(w) + + print r['/freebase/type_profile/published']['connect'] diff --git a/examples/freebase-images-appengine/freebase/rison.py b/examples/freebase-images-appengine/freebase/rison.py new file mode 100755 index 0000000..2f4eebc --- /dev/null +++ b/examples/freebase-images-appengine/freebase/rison.py @@ -0,0 +1,308 @@ +# +# rison for python (parser only so far) +# see http://mjtemplate.org/examples/rison.html for more info +# + +###################################################################### +# +# the rison parser is based on javascript openlaszlo-json: +# Author: Oliver Steele +# Copyright: Copyright 2006 Oliver Steele. All rights reserved. +# Homepage: http:#osteele.com/sources/openlaszlo/json +# License: MIT License. +# Version: 1.0 +# + +# hacked by nix for use in uris +# ported to python by nix +# +# TODO +# +# switch to unicode +# fall through to simplejson if first char is not in '!(' - +# this allows code to use just one parser +# + + +import os, sys, re +#import simplejson +simplejson = None + +class ParserException(Exception): + pass + +class Parser(object): + WHITESPACE = '' + #WHITESPACE = " \t\n\r\f" + + # we divide the uri-safe glyphs into three sets + # and classes are illegal in ids. + # - used by rison (possibly later) + # - not common in strings, reserved + #not_idchar = "'!=:(),*@$;&"; + + idchar_punctuation = '_-./~' + not_idchar = ''.join([c for c in (chr(i) for i in range(127)) + if not (c.isalnum() + or c in idchar_punctuation)]) + + # additionally, we need to distinguish ids and numbers by first char + not_idstart = "-0123456789"; + + # regexp string matching a valid id + idrx = ('[^' + not_idstart + not_idchar + + '][^' + not_idchar + ']*') + + # regexp to check for valid rison ids + id_ok_re = re.compile('^' + idrx + '$', re.M) + + # regexp to find the end of an id when parsing + next_id_re = re.compile(idrx, re.M) + + def parse_json(self, str): + if len(str) > 0 and str[0] not in '!(': + return simplejson.loads(str) + return self.parse(str) + + def parse(self, str): + self.string = str + self.index = 0 + + value = self.readValue() + if self.next(): + raise ParserException("unable to parse rison string %r" % (str,)) + return value + + def readValue(self): + c = self.next() + + if c == '!': + return self.parse_bang() + if c == '(': + return self.parse_open_paren() + if c == "'": + return self.parse_single_quote() + if c in '-0123456789': + return self.parse_number() + + # fell through table, parse as an id + s = self.string + i = self.index-1 + + m = self.next_id_re.match(s, i) + if m: + id = m.group(0) + self.index = i + len(id) + return id # a string + + if c: + raise ParserException("invalid character: '" + c + "'") + raise ParserException("empty expression") + + def parse_array(self): + ar = [] + while 1: + c = self.next() + if c == ')': + return ar + + if c is None: + raise ParserException("unmatched '!('") + + if len(ar): + if c != ',': + raise ParserException("missing ','") + elif c == ',': + raise ParserException("extra ','") + else: + self.index -= 1 + n = self.readValue() + ar.append(n) + + return ar + + def parse_bang (self): + s = self.string + c = s[self.index] + self.index += 1 + if c is None: + raise ParserException('"!" at end of input') + if c not in self.bangs: + raise ParserException('unknown literal: "!' + c + '"') + x = self.bangs[c] + if callable(x): + return x(self) + + return x + + + def parse_open_paren (self): + count = 0 + o = {} + + while 1: + c = self.next() + if c == ')': + return o + if count: + if c != ',': + raise ParserException("missing ','") + elif c == ',': + raise ParserException("extra ','") + else: + self.index -= 1 + k = self.readValue() + + if self.next() != ':': + raise ParserException("missing ':'") + v = self.readValue() + + o[k] = v + count += 1 + + + def parse_single_quote(self): + s = self.string + i = self.index + start = i + segments = [] + + while 1: + if i >= len(s): + raise ParserException('unmatched "\'"') + + c = s[i] + i += 1 + if c == "'": + break + + if c == '!': + if start < i-1: + segments.append(s[start:i-1]) + c = s[i] + i += 1 + if c in "!'": + segments.append(c) + else: + raise ParserException('invalid string escape: "!'+c+'"') + + start = i + + + if start < i-1: + segments.append(s[start:i-1]) + self.index = i + return ''.join(segments) + + + # Also any number start (digit or '-') + def parse_number(self): + s = self.string + i = self.index + start = i-1 + state = 'int' + permittedSigns = '-' + transitions = { + 'int+.': 'frac', + 'int+e': 'exp', + 'frac+e': 'exp' + } + + while 1: + if i >= len(s): + i += 1 + break + + c = s[i] + i += 1 + + if '0' <= c and c <= '9': + continue + + if permittedSigns.find(c) >= 0: + permittedSigns = '' + continue + + state = transitions.get(state + '+' + c.lower(), None) + if state is None: + break + if state == 'exp': + permittedSigns = '-' + + self.index = i - 1 + s = s[start:self.index] + if s == '-': + raise ParserException("invalid number") + if re.search('[.e]', s): + return float(s) + return int(s) + + # return the next non-whitespace character, or undefined + def next(self): + l = len(self.string) + s = self.string + i = self.index + + while 1: + if i == len(s): + return None + c = s[i] + i += 1 + if c not in self.WHITESPACE: + break + + self.index = i + return c + + + bangs = { + 't': True, + 'f': False, + 'n': None, + '(': parse_array + } + + +def loads(s): + return Parser().parse(s) + +if __name__ == '__main__': + p = Parser() + + rison_examples = [ + "(a:0,b:1)", + "(a:0,b:foo,c:'23skidoo')", + "!t", + "!f", + "!n", + "''", + "0", + "1.5", + "-3", + "1e30", + "1e-30", + "G.", + "a", + "'0a'", + "'abc def'", + "()", + "(a:0)", + "(id:!n,type:/common/document)", + "!()", + "!(!t,!f,!n,'')", + "'-h'", + "a-z", + "'wow!!'", + "domain.com", + "'user@domain.com'", + "'US $10'", + "'can!'t'", + ]; + + for s in rison_examples: + print + print '*'*70 + print + print s + + print '%r' % (p.parse(s),) diff --git a/examples/freebase-images-appengine/freebase/sandbox.py b/examples/freebase-images-appengine/freebase/sandbox.py new file mode 100644 index 0000000..e482118 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/sandbox.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# ======================================================================== +# Copyright (c) 2007, Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ======================================================================== + +import sys +from freebase.api.session import HTTPMetawebSession + +_base = HTTPMetawebSession("sandbox.freebase.com") + +__all__ = ["HTTPMetawebSession"] + +# we want to add base's functions to __init__.py +# so that we can say freebase.func() and really +# just call base.func() + +# a little trick to refer to ourselves +self = sys.modules[__name__] + +for funcname in dir(_base): + + # we only want the 'real' functions + if not funcname.startswith("_"): + func = getattr(_base, funcname) + + # let's make sure we're getting functions + # instead of constants or whatever + if callable(func): + + # we're setting these functions + # so that they can be called like + # freebase.funcname -> base.func() + setattr(self, funcname, func) + + # make sure we import the base's + # functions if we import freebase + __all__.append(funcname) + +# we don't want any self-referencing +# business going. Plus, this is cleaner. +del self + +# we want dir(freebase) to be clean +del funcname, func diff --git a/examples/freebase-images-appengine/freebase/sandbox.pyc b/examples/freebase-images-appengine/freebase/sandbox.pyc new file mode 100644 index 0000000..4f8ad46 Binary files /dev/null and b/examples/freebase-images-appengine/freebase/sandbox.pyc differ diff --git a/examples/freebase-images-appengine/freebase/uritemplate.py b/examples/freebase-images-appengine/freebase/uritemplate.py new file mode 100755 index 0000000..0a81147 --- /dev/null +++ b/examples/freebase-images-appengine/freebase/uritemplate.py @@ -0,0 +1,143 @@ +# ======================================================================== +# Copyright (c) 2007, Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ======================================================================== + +# +# URI Templating in Python +# +# see http://bitworking.org/projects/URI-Templates/ +# and http://bitworking.org/news/URI_Templates +# +# note that this implementation may go away soon in +# favor of joe gregorio's own code: +# http://code.google.com/p/uri-templates/ +# +# +# this implementation can also parse URIs, as long as the +# template is sufficiently specific. to allow this to work +# the '/' character is forbidden in keys when parsing. +# later it should be possible to loosen this restriction. +# +# +# example: +# from whatever.uritemplate import expand_uri_template +# expand_uri_template('http://{host}/{file}', +# dict(host='example.org', +# file='fred')) +# +# TODO: +# allow parsing to be aware of http://www. and trailing / +# nail down quoting issues +# + +import os, sys, re +import urllib + +__all__ = ['expand_uri_template', 'URITemplate'] + + +def expand_uri_template(template, args): + """Expand a URI template using the given args dictionary. + """ + return URITemplate(template).run(args) + +def _uri_encode_var(v): + return urllib.quote(v, safe="-_.~!$&'()*+,;=:/?[]#@") + + +class URITemplate(object): + """a URITemplate is a URI with simple variable substitution. + """ + + VARREF = re.compile(r'\{([0-9a-zA-Z_]+)\}') + + def __init__(self, s): + """Compile a URITemplate from a string. + """ + self.template = s; + + self.params = [] + tsplit = self.VARREF.split(s) + rxs = ['^'] + for i in range(len(tsplit)): + if i % 2: + # track the vars used + self.params.append(tsplit[i]) + # vars match any string + # vars are assumed to lack '/' - this is imperfect... + rxs.append('([^/]*)') + else: + # quote special chars regexp interpretation + rxs.append(re.escape(tsplit[i])) + rxs.append('$') + self._parser = re.compile(''.join(rxs)) + + def __repr__(self): + return '' % self.template + + def run (self, args): + """Expand the template using the given args. + """ + def repl(m): + key = m.group(1) + return _uri_encode_var(args.get(key, '')) + uri = self.VARREF.sub(repl,self.template) + + #if self.parse(uri) is None: + # print 're-parsing generated uri failed: %r, %r' % (uri, self.template) + return uri + + + def parse(self, uri): + """Try to parse a URI, extracting variable values. + """ + m = self._parser.match(uri) + if m is None: + return None + return dict(zip(self.params, m.groups())) + + +if __name__ == '__main__': + # + # testcases are imported from the URI::Template module on CPAN + # + import urllib2, simplejson + fp = urllib2.urlopen('http://search.cpan.org/src/BRICAS/URI-Template-0.09/t/data/spec.json') + testsuite = simplejson.loads(fp.read()) + vars = dict([(k.encode('utf-8'),v) for k,v in testsuite['variables'].items()]) + nsucceed = 0 + nfail = 0 + for test in testsuite['tests']: + ut = URITemplate(test['template']) + uri = ut.run(vars) + if uri != test['expected']: + print 'FAILED %r expected %r' % (uri, test['expected']) + print ' vars: %r' % (vars,) + nfail += 1 + else: + nsucceed += 1 + print 'tests: %d succeeded, %d failed' % (nsucceed, nfail) diff --git a/examples/freebase-images-appengine/images.html b/examples/freebase-images-appengine/images.html deleted file mode 100644 index 9688822..0000000 --- a/examples/freebase-images-appengine/images.html +++ /dev/null @@ -1,3 +0,0 @@ -{% for im in images %} - -{% endfor %} \ No newline at end of file -- cgit v1.3.1