reorganize package inorder to emphasize freebase-api as the main one

git-svn-id: http://freebase-python.googlecode.com/svn/trunk@88 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2

16 files changed, 3170 insertions, 0 deletions

diff --git a/freebase/__init__.py b/freebase/__init__.py
new file mode 100755
index 0000000..3cca4ff
--- /dev/null
+++ b/freebase/__init__.py
@@ -0,0 +1,40 @@
+import sys
+
+from freebase.api.session import HTTPMetawebSession
+import sandbox
+
+__all__ = ["HTTPMetawebSession", "sandbox"]
+
+base = HTTPMetawebSession("freebase.com")
+
+# we want to add base's functions to __init__.py
+# so that we can say freebase.func() and really
+# just call base.func()
+
+# a little trick to refer to __init__
+# self isn't defined because __init__ is in
+# a world in and of itself 
+self = sys.modules[__name__]
+
+for funcname in dir(base):
+    
+    # we only want the 'real' functions
+    if not funcname.startswith("_"):
+        func = getattr(base, funcname)
+        
+        # let's make sure we're getting functions
+        # instead of constants or whatever
+        if callable(func):
+            
+            # we're setting these functions
+            # so that they can be called like
+            # freebase.funcname -> base.func()
+            setattr(self, funcname, func)
+            
+            # make sure we import the base's
+            # functions if we import freebase
+            __all__.append(funcname)
+
+# we don't want any self-referencing
+# business going. Plus, this is cleaner.
+del self
diff --git a/freebase/api/__init__.py b/freebase/api/__init__.py
new file mode 100755
index 0000000..c3adebc
--- /dev/null
+++ b/freebase/api/__init__.py
@@ -0,0 +1,4 @@
+
+from session import HTTPMetawebSession, MetawebError, attrdict
+
+from mqlkey import quotekey, unquotekey
diff --git a/freebase/api/cookie_handlers.py b/freebase/api/cookie_handlers.py
new file mode 100755
index 0000000..88f6b4f
--- /dev/null
+++ b/freebase/api/cookie_handlers.py
@@ -0,0 +1,215 @@
+# ========================================================================
+# Copyright (c) 2007, Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ========================================================================
+
+#
+#
+#  httplib2cookie.py allows you to use python's standard
+#   CookieJar class with httplib2.
+#
+#
+
+import re
+
+try:
+    from google.appengine.api import urlfetch
+    Http = object
+except ImportError:
+    pass
+
+try:
+    from httplib2 import Http
+except ImportError:
+    pass
+
+try:
+    import urllib
+except ImportError:
+    import urllib_stub as urllib
+import cookielib
+
+class DummyRequest(object):
+    """Simulated urllib2.Request object for httplib2
+
+       implements only what's necessary for cookielib.CookieJar to work
+    """
+    def __init__(self, url, headers=None):
+        self.url = url
+        self.headers = headers
+        self.origin_req_host = cookielib.request_host(self)
+        self.type, r = urllib.splittype(url)
+        self.host, r = urllib.splithost(r)
+        if self.host:
+            self.host = urllib.unquote(self.host)
+
+    def get_full_url(self):
+        return self.url
+
+    def get_origin_req_host(self):
+        # TODO to match urllib2 this should be different for redirects
+        return self.origin_req_host
+
+    def get_type(self):
+        return self.type
+
+    def get_host(self):
+        return self.host
+
+    def get_header(self, key, default=None):
+        return self.headers.get(key.lower(), default)
+
+    def has_header(self, key):
+        return key in self.headers
+
+    def add_unredirected_header(self, key, val):
+        # TODO this header should not be sent on redirect
+        self.headers[key.lower()] = val
+
+    def is_unverifiable(self):
+        # TODO to match urllib2, this should be set to True when the
+        #  request is the result of a redirect
+        return False
+
+class DummyHttplib2Response(object):
+    """Simulated urllib2.Request object for httplib2
+
+       implements only what's necessary for cookielib.CookieJar to work
+    """
+    def __init__(self, response):
+        self.response = response
+
+    def info(self):
+        return DummyHttplib2Message(self.response)
+
+
+class DummyUrlfetchResponse(object):
+    """Simulated urllib2.Request object for httplib2
+
+       implements only what's necessary for cookielib.CookieJar to work
+    """
+    def __init__(self, response):
+        self.response = response
+
+    def info(self):
+        return DummyUrlfetchMessage(self.response)
+
+
+class DummyHttplib2Message(object):
+    """Simulated mimetools.Message object for httplib2
+
+       implements only what's necessary for cookielib.CookieJar to work
+    """
+    def __init__(self, response):
+        self.response = response
+
+    def getheaders(self, k):
+        k = k.lower()
+        v = self.response.get(k.lower(), None)
+        if k not in self.response:
+            return []
+        #return self.response[k].split(re.compile(',\\s*'))
+
+        # httplib2 joins multiple values for the same header
+        #  using ','.  but the netscape cookie format uses ','
+        #  as part of the expires= date format.  so we have
+        #  to split carefully here - header.split(',') won't do it.
+        HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)')
+        return [h[0] for h in HEADERVAL.findall(self.response[k])]
+
+class DummyUrlfetchMessage(object):
+    """Simulated mimetools.Message object for httplib2
+
+       implements only what's necessary for cookielib.CookieJar to work
+    """
+    def __init__(self, response):
+        self.response = response
+
+    def getheaders(self, k):
+        k = k.lower()
+        v = self.response.headers.get(k.lower(), None)
+        if k not in self.response.headers:
+            return []
+        #return self.response[k].split(re.compile(',\\s*'))
+
+        # httplib2 joins multiple values for the same header
+        #  using ','.  but the netscape cookie format uses ','
+        #  as part of the expires= date format.  so we have
+        #  to split carefully here - header.split(',') won't do it.
+        HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)')
+        return [h[0] for h in HEADERVAL.findall(self.response.headers[k])]
+
+class CookiefulHttp(Http):
+    """Subclass of httplib2.Http that keeps cookie state
+
+       constructor takes an optional cookiejar=cookielib.CookieJar 
+
+       currently this does not handle redirects completely correctly:
+       if the server redirects to a different host the original
+       cookies will still be sent to that host.
+    """
+    def __init__(self, cookiejar=None, **kws):
+        # note that httplib2.Http is not a new-style-class
+        Http.__init__(self, **kws)
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def request(self, uri, **kws):
+        headers = kws.pop('headers', None)
+        req = DummyRequest(uri, headers)
+        self.cookiejar.add_cookie_header(req)
+        headers = req.headers
+
+        (r, body) = Http.request(self, uri, headers=headers, **kws)
+
+        resp = DummyHttplib2Response(r)
+        self.cookiejar.extract_cookies(resp, req)
+
+        return (r, body)
+
+class CookiefulUrlfetch(object):
+    """Class that keeps cookie state
+
+       constructor takes an optional cookiejar=cookielib.CookieJar
+    """ 
+    # TODO refactor CookefulHttp so that CookiefulUrlfetch can be a subclass of it
+    def __init__(self, cookiejar=None, **kws):
+        if cookiejar is None:
+            cookejar = cookielib.CookieJar()
+        self.cookejar = cookiejar
+
+    def request(self, uri, **kws):
+        headers = kws.pop('headers', None)
+        req = DummyRequest(uri, headers)
+        self.cookejar.add_cookie_header(req)
+        headers = req.headers
+
+        r = urlfetch.fetch(uri, headers=headers, **kws)
+
+        self.cookejar.extract_cookies(DummyUrlfetchResponse(r), req)
+        return r
+
diff --git a/freebase/api/httpclients.py b/freebase/api/httpclients.py
new file mode 100644
index 0000000..48af9b6
--- /dev/null
+++ b/freebase/api/httpclients.py
@@ -0,0 +1,91 @@
+try:
+    from google.appengine.api import urlfetch
+    from cookie_handlers import CookiefulUrlfetch
+except:
+    pass
+
+try:
+    import httplib2
+    from cookie_handlers import CookiefulHttp
+except:
+    pass
+
+try:
+    import urllib2
+    import socket
+except:
+    pass
+
+import logging
+import re
+
+class Urllib2Client(object):
+    def __init__(self, cookiejar, rse):
+        cookiespy = urllib2.HTTPCookieProcessor(cookiejar)
+        self.opener = urllib2.build_opener(cookiespy)
+        self._raise_service_error = rse
+        self.log = logging.getLogger()
+
+    def __call__(self, url, method, body, headers):
+        req = urllib2.Request(url, body, headers)
+
+        try:
+            resp = self.opener.open(req)
+
+        except socket.error, e:
+            self.log.error('SOCKET FAILURE: %s', e.fp.read())
+            raise MetawebError, 'failed contacting %s: %s' % (url, str(e))
+
+        except urllib2.HTTPError, e:
+            self.log.error('HTTP ERROR: %s', e)
+            self._raise_service_error(url, e.code, e.info().type, e.fp.read())
+
+        for header in resp.info().headers:
+            self.log.debug('HTTP HEADER %s', header)
+            name, value = re.split("[:\n\r]", header, 1)
+            if name.lower() == 'x-metaweb-tid':
+                self.tid = value.strip()
+
+        return (resp, resp.read())
+
+class Httplib2Client(object):
+    def __init__(self, cookiejar, rse):
+        self.cookiejar = cookiejar
+        self._raise_service_error = rse
+        self.httpclient = CookiefulHttp(cookiejar=self.cookiejar)
+
+    def __call__(self, url, method, body, headers):
+        try:
+            resp, content = self.httpclient.request(url, method=method,
+                                                    body=body, headers=headers)
+            if (resp.status != 200):
+                self._raise_service_error(url, resp.status, resp['content-type'], content)
+
+        except socket.error, e:
+            self.log.error('SOCKET FAILURE: %s', e.fp.read())
+            raise MetawebError, 'failed contacting %s: %s' % (url, str(e))
+
+        except httplib2.HttpLib2ErrorWithResponse, e:
+            self._raise_service_error(url, resp.status, resp['content-type'], content)
+        except httplib2.HttpLib2Error, e:
+            raise MetawebError(u'HTTP error: %s' % (e,))
+
+        #tid = resp.get('x-metaweb-tid', None)
+
+        return (resp, content)
+
+
+class UrlfetchClient(object):
+    def __init__(self, cookiejar, rse):
+        self.cookiejar = cookiejar
+        self._raise_service_error = rse
+        self.httpclient = CookiefulUrlfetch(cookiejar=self.cookiejar)
+
+    def __call__(self, url, method, body, headers):
+        resp = self.httpclient.request(url, payload=body, method=method, headers=headers)
+
+        if resp.status_code != 200:
+            self._raise_service_error(url, resp.status_code, resp.headers['content-type'], resp.body)
+
+        return (resp, resp.content)
+
diff --git a/freebase/api/mqlkey.py b/freebase/api/mqlkey.py
new file mode 100755
index 0000000..d7dd01b
--- /dev/null
+++ b/freebase/api/mqlkey.py
@@ -0,0 +1,135 @@
+# ========================================================================
+# Copyright (c) 2007, Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ========================================================================
+
+import string
+import re
+
+def quotekey(ustr):
+    """
+    quote a unicode string to turn it into a valid namespace key
+    
+    """
+    valid_always = string.ascii_letters + string.digits
+    valid_interior_only = valid_always + '_-'
+
+    if isinstance(ustr, str):
+        s = unicode(ustr,'utf-8')        
+    elif isinstance(ustr, unicode):
+        s = ustr
+    else:
+        raise ValueError, 'quotekey() expects utf-8 string or unicode'
+
+    output = []
+    if s[0] in valid_always:
+        output.append(s[0])
+    else:
+        output.append('$%04X' % ord(s[0]))
+
+    for c in s[1:-1]:
+        if c in valid_interior_only:
+            output.append(c)
+        else:
+            output.append('$%04X' % ord(c))
+
+    if len(s) > 1:
+        if s[-1] in valid_always:
+            output.append(s[-1])
+        else:
+            output.append('$%04X' % ord(s[-1]))
+
+    return str(''.join(output))
+
+
+def unquotekey(key, encoding=None):
+    """
+    unquote a namespace key and turn it into a unicode string
+    """
+
+    valid_always = string.ascii_letters + string.digits
+
+    output = []
+    i = 0
+    while i < len(key):
+        if key[i] in valid_always:
+            output.append(key[i])
+            i += 1
+        elif key[i] in '_-' and i != 0 and i != len(key):
+            output.append(key[i])
+            i += 1
+        elif key[i] == '$' and i+4 < len(key):
+            # may raise ValueError if there are invalid characters
+            output.append(unichr(int(key[i+1:i+5],16)))
+            i += 5
+        else:
+            raise ValueError, "unquote key saw invalid character '%s' at position %d" % (key[i], i)
+
+    ustr = u''.join(output)
+    
+    if encoding is None:
+        return ustr
+
+    return ustr.encode(encoding)
+
+
+# should this also include "'()" into safe?
+def urlencode_pathseg(data):
+    '''
+    urlencode for placement between slashes in an url.
+    '''
+    if isinstance(data, unicode):
+        data = data.encode('utf_8')
+    return urllib.quote(data, "~:@$!*,;=&+")
+
+
+def id_to_urlid(id):
+    """
+    convert a mql id to an id suitable for embedding in a url path.
+    """
+
+    segs = id.split('/')
+
+    assert isinstance(id, str) and id != '', 'bad id "%s"' % id
+
+    if id[0] == '~':
+        assert len(segs) == 1
+        # assume valid, should check
+        return id
+
+    if id[0] == '#':
+        assert len(segs) == 1
+        # assume valid, should check
+        return '%23' + id[1:]
+
+    if id[0] != '/':
+        raise ValueError, 'unknown id format %s' % id
+
+    # ok, we have a slash-path
+    # requote components as keys and rejoin.
+    # urlids do not have leading slashes!!!
+    return '/'.join(urlencode_pathseg(unquotekey(seg)) for seg in segs[1:])
+
diff --git a/freebase/api/session.py b/freebase/api/session.py
new file mode 100644
index 0000000..0d5159b
--- /dev/null
+++ b/freebase/api/session.py
@@ -0,0 +1,687 @@
+# ==================================================================
+# Copyright (c) 2007, Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ====================================================================
+
+"""
+declarations for external metaweb api.
+
+
+    from metaweb.api import HTTPMetawebSession
+
+    mss = HTTPMetawebSession('sandbox.freebase.com')
+    print mss.mqlread([dict(name=None, type='/type/type')])
+"""
+
+
+
+__all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict']
+__version__ = '0.1'
+
+import os, sys, re
+import cookielib
+try:
+    import simplejson
+except ImportError:
+    try:
+        # appengine provides simplejson at django.utils.simplejson
+        from django.utils import simplejson
+    except ImportError:
+        raise Exception("unable to import simplejson")
+try:
+    from urllib import quote as urlquote
+except ImportError:
+    from urlib_stub import quote as urlquote
+import pprint
+import socket
+import logging
+
+class Delayed(object):
+    """
+    Wrapper for callables in log statements. Avoids actually making
+    the call until the result is turned into a string.
+
+    A few examples:
+
+    simplejson.dumps is never called because the logger never
+    tries to format the result
+    >>> logging.debug(Delayed(simplejson.dumps, q))
+
+    This time simplejson.dumps() is actually called:
+    >>> logging.warn(Delayed(simplejson.dumps, q))
+    
+    """
+    def __init__(self, f, *args, **kwds):
+        self.f = f
+        self.args = args
+        self.kwds = kwds
+
+    def __str__(self):
+        return str(self.f(*self.args, **self.kwds))
+    
+def logformat(result):
+    """
+    Format the dict/list as a json object
+    """
+    rstr = simplejson.dumps(result, indent=2)
+    if rstr[0] == '{':
+        rstr = rstr[1:-2]
+    return rstr
+
+from httpclients import Httplib2Client, Urllib2Client, UrlfetchClient
+
+# Check for urlfetch first so that urlfetch is used when running the appengine SDK
+try:
+    import google.appengine.api.urlfetch
+    from cookie_handlers import CookiefulUrlfetch
+    http_client = UrlfetchClient
+except ImportError:
+    try:
+        import httplib2
+        from cookie_handlers import CookiefulHttp
+        http_client = Httplib2Client
+    except ImportError:
+        import urllib2
+        httplib2 = None
+        CookiefulHttp = None
+        http_client = Urllib2Client
+
+# remove whitespace from json encoded output
+simplejson.JSONEncoder.item_separator = ','
+simplejson.JSONEncoder.key_separator = ':'
+# don't escape slashes, we're not pasting into script tags here.
+if simplejson.encoder.ESCAPE_DCT.get('/', None) == r'\/':
+    simplejson.encoder.ESCAPE_DCT['/'] = '/'
+
+def urlencode_weak(s):
+    return urlquote(s, safe=',/:$')
+
+
+# from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/361668
+class attrdict(dict):
+    """A dict whose items can also be accessed as member variables.
+
+    >>> d = attrdict(a=1, b=2)
+    >>> d['c'] = 3
+    >>> print d.a, d.b, d.c
+    1 2 3
+    >>> d.b = 10
+    >>> print d['b']
+    10
+
+    # but be careful, it's easy to hide methods
+    >>> print d.get('c')
+    3
+    >>> d['get'] = 4
+    >>> print d.get('a')
+    Traceback (most recent call last):
+    TypeError: 'int' object is not callable
+    """
+    def __init__(self, *args, **kwargs):
+        dict.__init__(self, *args, **kwargs)
+        self.__dict__ = self
+
+
+
+# TODO expose the common parts of the result envelope
+class MetawebError(Exception):
+    """
+    an error report from the metaweb service.
+    """
+    pass
+
+
+
+# TODO right now this is a completely unnecessary superclass.
+#  is there enough common behavior between session types
+#  to justify it?
+class MetawebSession(object):
+    """
+    MetawebSession is the base class for MetawebSession, subclassed for
+    different connection types.  Only http is available externally.
+
+    This is more of an interface than a class
+    """
+
+    # interface definition here...
+    
+
+# from httplib2
+NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+')
+def _normalize_headers(headers):
+    return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip())  for (key, value) in headers.iteritems()])
+
+class HTTPMetawebSession(MetawebSession):
+    """
+    a MetawebSession is a request/response queue.
+
+    this version uses the HTTP api, and is synchronous.
+    """
+    # share cookies across sessions, so that different sessions can
+    #  see each other's writes immediately.
+    _default_cookiejar = cookielib.CookieJar()
+
+    def __init__(self, service_url, username=None, password=None, prev_session=None, cookiejar=None, cookiefile=None):
+        """
+        create a new MetawebSession for interacting with the Metaweb.
+
+        a new session will inherit state from prev_session if present, 
+        """
+        super(HTTPMetawebSession, self).__init__()
+
+        self.log = logging.getLogger()
+
+        assert not service_url.endswith('/')
+        if not '/' in service_url:  # plain host:port
+            service_url = 'http://' + service_url
+
+        self.service_url = service_url
+
+        self.username = username
+        self.password = password
+
+        self.tid = None
+
+        if prev_session:
+            self.service_url = prev.service_url
+
+        if cookiefile is not None:
+            cookiejar = self.open_cookie_file(cookiefile)
+
+        if cookiejar is not None:
+            self.cookiejar = cookiejar
+        elif prev_session:
+            self.cookiejar = prev_session.cookiejar
+        else:
+            self.cookiejar = self._default_cookiejar
+
+        self._http_request = http_client(self.cookiejar, self._raise_service_error)
+
+
+    def open_cookie_file(self, cookiefile=None):
+        if cookiefile is None or cookiefile == '':
+            if os.environ.has_key('HOME'):
+                cookiefile = os.path.join(os.environ['HOME'], '.pyfreebase/cookiejar')
+            else:
+                raise MetawebError("no cookiefile specified and no $HOME/.pyfreebase directory" % cookiefile)
+
+        cookiejar = cookielib.LWPCookieJar(cookiefile)
+        if os.path.exists(cookiefile):
+            cookiejar.load(ignore_discard=True)
+
+        return cookiejar
+
+
+    def _httpreq(self, service_path, method='GET', body=None, form=None,
+                 headers=None):
+        """
+        make an http request to the service.
+
+        form arguments are encoded in the url, even for POST, if a non-form
+        content-type is given for the body.
+
+        returns a pair (resp, body)
+
+        resp is the response object and may be different depending
+        on whether urllib2 or httplib2 is in use?
+        """
+
+        if method == 'GET':
+            assert body is None
+        if method != "GET" and method != "POST":
+            assert 0, 'unknown method %s' % method
+
+        url = self.service_url + service_path
+
+        if headers is None:
+            headers = {}
+        else:
+            headers = _normalize_headers(headers)
+
+        # this is a lousy way to parse Content-Type, where is the library?
+        ct = headers.get('content-type', None)
+        if ct is not None:
+            ct = ct.split(';')[0]
+
+        if body is not None:
+            # if body is provided, content-type had better be too
+            assert ct is not None
+
+        if form is not None:
+            qstr = '&'.join(['%s=%s' % (urlencode_weak(unicode(k)), urlencode_weak(unicode(v)))
+                             for k,v in form.items()])
+            if method == 'POST':
+                # put the args on the url if we're putting something else
+                # in the body.  this is used to add args to raw uploads.
+                if body is not None:
+                    url += '?' + qstr
+                else:
+                    if ct is None:
+                        ct = 'application/x-www-form-urlencoded'
+                        headers['content-type'] = ct + '; charset=utf-8'
+
+                    if ct == 'multipart/form-encoded':
+                        # TODO handle this case
+                        raise NotImplementedError
+                    elif ct == 'application/x-www-form-urlencoded':
+                        body = qstr
+            else:
+                # for all methods other than POST, use the url
+                url += '?' + qstr
+
+
+        # assure the service that this isn't a CSRF form submission
+        headers['x-metaweb-request'] = 'Python'
+
+        if 'user-agent' not in headers:
+            headers['user-agent'] = 'python freebase.api-%s' % __version__
+
+        #if self.tid is not None:
+        #    headers['x-metaweb-tid'] = self.tid
+
+        ####### DEBUG MESSAGE - should check log level before generating 
+        if form is None:
+            formstr = ''
+        else:
+            formstr = '\nFORM:\n  ' + '\n  '.join(['%s=%s' % (k,v)
+                                          for k,v in form.items()])
+        if headers is None:
+            headerstr = ''
+        else:
+            headerstr = '\nHEADERS:\n  ' + '\n  '.join([('%s: %s' % (k,v))
+                                              for k,v in headers.items()])
+        self.log.info('%s %s%s%s', method, url, formstr, headerstr)
+        #######
+
+        return self._http_request(url, method, body, headers)
+
+    def _raise_service_error(self, url, status, ctype, body):
+
+        is_jsbody = (ctype.endswith('javascript')
+                     or ctype.endswith('json'))
+        if str(status) == '400' and is_jsbody:
+            r = self._loadjson(body)
+            msg = r.messages[0]
+            raise MetawebError(u'%s %s %r' % (msg.get('code',''), msg.message, msg.info))
+
+        raise MetawebError, 'request failed: %s: %r %r' % (url, status, body)
+        
+    def _httpreq_json(self, *args, **kws):
+        resp, body = self._httpreq(*args, **kws)
+        return self._loadjson(body)
+
+    def _loadjson(self, json):
+        # TODO really this should be accomplished by hooking
+        # simplejson to create attrdicts instead of dicts.
+        def struct2attrdict(st):
+            """
+            copy a json structure, turning all dicts into attrdicts.
+            
+            copying descends instances of dict and list, including subclasses.
+            """
+            if isinstance(st, dict):
+                return attrdict([(k,struct2attrdict(v)) for k,v in st.items()])
+            if isinstance(st, list):
+                return [struct2attrdict(li) for li in st]
+            return st
+
+        if json == '':
+            self.log.error('the empty string is not valid json')
+            raise MetawebError('the empty string is not valid json')
+
+        try:
+            r = simplejson.loads(json)
+        except ValueError, e:
+            self.log.error('error parsing json string %r' % json)
+            raise MetawebError, 'error parsing JSON string: %s' % e
+
+        return struct2attrdict(r)
+
+    def _check_mqlerror(self, r):
+        if r.code != '/api/status/ok':
+            for msg in r.messages:
+                self.log.error('mql error: %s %s %r' % (msg.code, msg.message, msg.get('query', None)))
+            raise MetawebError, 'query failed: %s\n%r' % (r.messages[0].code, r.messages[0].get('query', None))
+
+    def _mqlresult(self, r):
+        self._check_mqlerror(r)
+
+        self.log.info('result: %s', Delayed(logformat, r))
+
+        return r.result
+
+
+
+    def login(self, username=None, password=None):
+        """sign in to the service. For a more complete description,
+        see http://www.freebase.com/view/en/api_account_login"""
+
+        service = '/api/account/login'
+
+        username = username or self.username
+        password = password or self.password
+
+        assert username is not None
+        assert password is not None
+        
+        self.log.debug('LOGIN USERNAME: %s', username)
+        
+        r = self._httpreq_json(service, 'POST',
+                               form=dict(username=username,
+                                         password=password))
+
+        if r.code != '/api/status/ok':
+            raise MetawebError(u'%s %r' % (r.get('code',''), r.messages))
+
+        self.log.debug('LOGIN RESP: %r', r)
+        self.log.debug('LOGIN COOKIES: %s', self.cookiejar)
+
+    def logout(self):
+        """logout of the service. For a more complete description,
+        see http://www.freebase.com/view/en/api_account_logout"""
+        
+        service = '/api/account/logout'
+        
+        self.log.debug("LOGOUT")
+        
+        r = self._httpreq_json(service, 'GET')
+        
+        if r.code != '/api/status/ok':
+            raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) #this should never happen
+
+    def user_info(self, mql_output=None):
+        """ get user_info. For a more complete description,
+        see http://www.freebase.com/view/en/api_service_user_info"""
+        
+        service = "/api/service/user_info"
+        
+        qstr = simplejson.dumps(mql_output)
+        
+        r = self._httpreq_json(service, 'POST', form=dict(mql_output=qstr))
+        return r
+    
+    def loggedin(self):
+        """check to see whether a user is logged in or not. For a
+        more complete description, see http://www.freebase.com/view/en/api_account_loggedin"""
+        
+        service = "/api/account/loggedin"
+        try:
+            r = self._httpreq_json(service, 'GET')
+            if r.code == "/api/status/ok":
+                return True
+                
+        except MetawebError, me:
+            return False
+        
+    def mqlreaditer(self, sq, asof=None):
+        """read a structure query."""
+
+        cursor = True
+
+        while 1:
+            subq = dict(query=[sq], cursor=cursor, escape=False)
+            if asof:
+                subq['as_of_time'] = asof
+
+            qstr = simplejson.dumps(subq)
+
+            service = '/api/service/mqlread'
+            
+            r = self._httpreq_json(service, form=dict(query=qstr))
+
+            for item in self._mqlresult(r):
+                yield item
+
+            if r['cursor']:
+                cursor = r['cursor']
+                self.log.info('CONTINUING with %s', cursor)
+            else:
+                return
+
+    def mqlread(self, sq, asof=None):
+        """read a structure query. For a more complete description,
+        see http://www.freebase.com/view/en/api_service_mqlread"""
+        subq = dict(query=sq, escape=False)
+        if asof:
+            subq['as_of_time'] = asof
+
+        if isinstance(sq, list):
+            subq['cursor'] = True
+
+        service = '/api/service/mqlread'
+
+        self.log.info('%s: %s',
+                      service,
+                      Delayed(logformat, sq))
+
+        qstr = simplejson.dumps(subq)
+        r = self._httpreq_json(service, form=dict(query=qstr))
+
+        return self._mqlresult(r)
+
+    def mqlreadmulti(self, queries, asof=None):
+        """read a structure query"""
+        keys = [('q%d' % i) for i,v in enumerate(queries)];
+        envelope = {}
+        for i,sq in enumerate(queries):
+            subq = dict(query=sq, escape=False)
+            if asof:
+                subq['as_of_time'] = asof
+
+            # XXX put this back once mqlreadmulti is working in general
+            #if isinstance(sq, list):
+            #    subq['cursor'] = True
+            envelope[keys[i]] = subq
+
+        service = '/api/service/mqlread'
+
+        self.log.info('%s: %s',
+                      service,
+                      Delayed(logformat, envelope))
+
+        qstr = simplejson.dumps(envelope)
+        rs = self._httpreq_json(service, form=dict(queries=qstr))
+
+        self.log.info('%s result: %s',
+                      service,
+                      Delayed(simplejson.dumps, rs, indent=2))
+
+        return [self._mqlresult(rs[key]) for key in keys]
+
+    def raw(self, id):
+        """translate blob from id. For a more complete description,
+        see http://www.freebase.com/view/en/api_trans_raw"""
+        url = '/api/trans/raw' + urlquote(id)
+
+        self.log.info(url)
+
+        resp, body = self._httpreq(url)
+
+        self.log.info('raw is %d bytes' % len(body))
+
+        return body
+    
+    def blurb(self, id, break_paragraphs=False, maxlength=200):
+        """translate only the text in blob from id. For a more 
+        complete description, see http://www.freebase.com/view/en/api_trans_blurb"""
+        url = '/api/trans/blurb' + urlquote(id)
+
+        self.log.info(url)
+
+        resp, body = self._httpreq(url, form=dict(break_paragraphs=break_paragraphs, maxlength=maxlength))
+
+        self.log.info('blurb is %d bytes' % len(body))
+
+        return body
+
+    def image_thumb(self, id, maxwidth=None, maxheight=None, mode="fit", onfail=None):
+        """ given the id of an image, this will return a URL of a thumbnail of the image.
+        The full details of how the image is cropped and finessed is detailed at 
+        http://www.freebase.com/view/en/api_trans_image_thumb """
+
+        service = "/api/trans/image_thumb"
+        assert mode in ["fit", "fill", "fillcrop", "fillcropmid"]
+        
+        form = dict(mode=mode)
+        if maxwidth is not None:
+            form["maxwidth"] = maxwidth
+        if maxheight is not None:
+            form["maxheight"] = maxheight
+        if onfail is not None:
+            form["onfail"] = onfail
+        
+        resp, body = self._httpreq(service + urlquote(id), form=form)
+        self.log.info('image is %d bytes' % len(body))
+        
+        return body
+
+    def mqlwrite(self, sq):
+        """do a mql write. For a more complete description,
+        see http://www.freebase.com/view/en/api_service_mqlwrite"""
+        query = dict(query=sq, escape=False)
+        qstr = simplejson.dumps(query)
+
+        self.log.debug('MQLWRITE: %s', qstr)
+
+        service = '/api/service/mqlwrite'
+
+        self.log.info('%s: %s',
+                      service,
+                      Delayed(logformat,sq))
+
+        r = self._httpreq_json(service, 'POST',
+                               form=dict(query=qstr))
+
+        self.log.debug('MQLWRITE RESP: %r', r)
+        return self._mqlresult(r)
+
+    def mqlcheck(self, sq):
+        """ See if a write is valid, and see what would happen, but do not
+        actually do the write """
+        
+        query = dict(query=sq, escape=False)
+        qstr = simplejson.dumps(query)
+
+        self.log.debug('MQLCHECK: %s', qstr)
+
+        service = '/api/service/mqlcheck'
+
+        self.log.info('%s: %s',
+                      service,
+                      Delayed(logformat, sq))
+
+        r = self._httpreq_json(service, 'POST',
+                               form=dict(query=qstr))
+
+
+        self.log.debug('MQLCHECK RESP: %r', r)
+
+        return self._mqlresult(r)
+    
+    def mqlflush(self):
+        """ask the service not to hand us old data"""
+        self.log.debug('MQLFLUSH')
+    
+        service = '/api/service/touch'
+        r = self._httpreq_json(service)
+
+        self._check_mqlerror(r)
+        return True
+
+    def touch(self):
+        """ make sure you are accessing the most recent data. For a more
+        complete description, see http://www.freebase.com/view/en/api_service_touch"""
+        return self.mqlflush()
+    
+
+    def upload(self, body, content_type, document_id=False, permission_of=False):
+        """upload to the metaweb. For a more complete description,
+        see http://www.freebase.com/view/en/api_service_upload"""
+
+        service = '/api/service/upload'
+
+        self.log.info('POST %s: %s (%d bytes)',
+                      service, content_type, len(body))
+
+
+        headers = {}
+        if content_type is not None:
+            headers['content-type'] = content_type
+
+        form = None
+        if document_id is not False:
+            if document_id is None:
+                form = { 'document': '' }
+            else:
+                form = { 'document': document_id }
+        if permission_of is not False:
+            if form:
+                form['permission_of'] = permission_of
+            else:
+                form = { 'permission_of' : permission_of }
+
+        # note the use of both body and form.
+        #  form parameters get encoded into the URL in this case
+        r = self._httpreq_json(service, 'POST',
+                               headers=headers, body=body, form=form)
+        return self._mqlresult(r)
+
+    
+    def version(self):
+        """ get versions for various parts of freebase. For a more
+        complete description, see http://www.freebase.com/view/en/api_version"""
+        
+        service = "/api/version"
+        r = self._httpreq_json(service)
+        
+        self._check_mqlerror(r)
+        return r
+    
+    ### DEPRECATED IN API
+    def reconcile(self, name, etype=['/common/topic']):
+        """DEPRECATED: reconcile name to guid. For a more complete description,
+        see http://www.freebase.com/view/en/dataserver_reconciliation"""
+
+        service = '/dataserver/reconciliation'
+        r = self._httpreq_json(service, 'GET', form={'name':name, 'types':','.join(etype)})
+
+        
+        # TODO non-conforming service, fix later
+        #self._mqlresult(r)
+        return r
+    
+
+if __name__ == '__main__':
+    console = logging.StreamHandler()
+    console.setLevel(logging.DEBUG)
+
+    mss = HTTPMetawebSession('sandbox.freebase.com')
+
+    self.mss.log.setLevel(logging.DEBUG)
+    self.mss.log.addHandler(console)
+
+
+    print mss.mqlread([dict(name=None, type='/type/type')])
diff --git a/freebase/fcl/README.txt b/freebase/fcl/README.txt
new file mode 100644
index 0000000..2e03d71
--- /dev/null
+++ b/freebase/fcl/README.txt
@@ -0,0 +1,8 @@
+
+This is a command-line tool for performing various freebase commands.
+
+It should be installed by default as "fcl".
+
+Try "fcl help" after installing the freebase-api library.
+
+
diff --git a/freebase/fcl/__init__.py b/freebase/fcl/__init__.py
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/freebase/fcl/__init__.py
diff --git a/freebase/fcl/commands.py b/freebase/fcl/commands.py
new file mode 100755
index 0000000..0fdc283
--- /dev/null
+++ b/freebase/fcl/commands.py
@@ -0,0 +1,688 @@
+# ==================================================================
+# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ====================================================================
+
+import os, sys, re, time
+from fbutil import *
+
+import simplejson
+
+import freebase.rison as rison
+from freebase.api import attrdict
+
+
+def cmd_help(fb, command=None):
+    """get help on commands
+
+    %prog help [cmd]
+    """
+
+    if command is not None:
+        cmd = fb.commands[command]
+        print '   %s: %s' % (cmd.name, cmd.shortdoc)
+        print cmd.doc
+        return
+
+    print """
+the interface to this tool is loosely modeled on the
+"svn" command-line tool.  many commands work slightly
+differently from their unix and svn equivalents - 
+read the doc for the command first.
+
+use "%s help <subcommand>" for help on a particular
+subcommand.
+""" % fb.progpath
+
+    fb.oparser.print_help()
+
+    print 'available subcommands:'
+    cmds = sorted(fb.commands.keys())
+
+    for fk in cmds:
+        cmd = fb.commands[fk]
+        print '   %s: %s' % (cmd.name, cmd.shortdoc)
+
+
+def cmd_wikihelp(fb):
+    """get help on commands in mediawiki markup format
+
+    %prog wikihelp
+    """
+
+    print """usage: %s subcommand [ args ... ]
+
+    the interface to this tool is loosely modeled on the
+    "svn" command-line tool.  many commands work slightly
+    differently from their unix and svn equivalents - 
+    read the doc for the command first.
+    """ % fb.progpath
+
+    print 'subcommands:'
+    for fk in sorted(fb.commands.keys()):
+        cmd = fb.commands[fk]
+
+        # mediawiki markup cares about the difference between
+        #  a blank line and a blank line with a bunch of
+        #  spaces at the start of it.  ewww.
+        doc = re.sub(r'\r?\n {0,8}', '\n        ', cmd.doc)
+
+        print '==== %s %s: %s ====' % (fb.progpath, cmd.name, cmd.shortdoc)
+        print doc
+        print
+
+
+# this command is disabled because it relies on some svn glue that
+# was not sufficiently well thought out.
+def old_cmd_pwid(fb):
+    """show the "current working namespace id"
+
+    %prog pwid
+
+    by default, relative ids can't be resolved.  however, if you
+    run the fb tool from a directory that has the svn property
+    'freebase:id' set, relative ids will be resolved relative
+    to that id.  the idea is that you can create an svn tree
+    that parallels the freebase namespace tree.  
+
+    for example:
+        $ %prog pwid
+
+        $ svn propset freebase:id "/freebase" .
+
+        $ %prog pwid
+        /freebase
+
+    """
+    if fb.cwid is not None:
+        print fb.cwid
+    else:
+        print ''
+
+
+def cmd_ls(fb, path=None):
+    """list the keys in a namespace
+
+    %prog ls [id]
+    """
+    path = fb.absid(path)
+    q = {'id': path,
+         '/type/namespace/keys': [{'value': None,
+                                   'namespace': {
+                                       'id': None,
+                                       'type': []
+                                    },
+                                    'optional':True
+                                   }]
+         }
+
+    r = fb.mss.mqlread(q)
+    if r is None:
+        raise CmdException('query for id %r failed' % path)
+
+    #sys.stdout.write(' '.join([mk.value for mk in r['/type/namespace/keys']]))
+
+    for mk in r['/type/namespace/keys']:
+        print mk.value
+
+    if 0:
+        suffix = ''
+        if ('/type/namespace' in mk.namespace.type
+            or '/type/domain' in mk.namespace.type):
+            suffix = '/'
+        print mk.value, mk.namespace.id+suffix
+
+
+def cmd_mkdir(fb, path):
+    """create a new freebase namespace
+    %prog mkdir id
+
+    create a new instance of /type/namespace at the given
+    point in id space.  if id already exists, it should be
+    a namespace.
+    """
+    path = fb.absid(path)
+    dir,file = dirsplit(path)
+    wq = { 'create': 'unless_exists',
+           'key':{
+               'connect': 'insert',
+               'namespace': dir,
+               'value': file
+               },
+           'name': path,
+           'type': '/type/namespace'
+         }
+
+    r = fb.mss.mqlwrite(wq)
+
+def cmd_ln(fb, src, dst):
+    """create a namespace key
+    %prog ln srcid dstid
+
+    create a new namespace link at dstid to the object
+    currently at srcid.
+    """
+    src = fb.absid(src)
+    dst = fb.absid(dst)
+    dir,file = dirsplit(dst)
+    wq = { 'id': src,
+           'key':{
+               'connect': 'insert',
+               'namespace': dir,
+               'value': file
+               }
+         }
+
+    r = fb.mss.mqlwrite(wq)
+
+
+def cmd_rm(fb, path):
+    """unlink a namespace key
+    %prog rm id
+    
+    remove the /type/key that connects the given id to its
+    parent.  id must be a path for this to make any sense.
+
+    note that is like unix 'unlink' rather than 'rm'.
+    it won't complain if the 'subdirectory' contains data,
+    since that data will still be accessible to other queries.
+    it's not like 'rm -rf' either, because it doesn't
+    disturb anything other than the one directory entry.
+    """
+    path = fb.absid(path)
+    dir,file = dirsplit(path)
+
+    wq = { 'id': path,
+           'key':{
+               'connect': 'delete',
+               'namespace': dir,
+               'value': file
+            }
+         }
+
+    r = fb.mss.mqlwrite(wq)
+
+def cmd_mv(fb, src, dst):
+    """rename srcid to dstid.
+    %prog mv srcid dstid
+
+    equivalent to:
+    $ fb ln <srcid> <dstid>
+    $ fb rm <srcid>
+    """
+    cmd_ln(fb, src, dst)
+    cmd_rm(fb, src)
+
+def cmd_cat(fb, id, include_headers=False):
+    """download a document from freebase to stdout
+    %prog cat id
+
+    equivalent to "%prog get id -".
+    """
+    return cmd_get(fb, id, localfile='-', include_headers=include_headers)
+
+def cmd_get(fb, id, localfile=None, include_headers=False):
+    """download a file from freebase
+    %prog get id [localfile]
+
+    download the document or image with the given id from freebase
+    into localfile.  localfile '-' means stdout.  localfile
+    defaults to a file in the current directory with the same name
+    as the last key in the path, possibly followed by a metadata
+    extension like .html or .txt.
+    """
+    id = fb.absid(id)
+    dir,file = dirsplit_unsafe(id)
+
+    def read_content(id, content_only=False):
+        c = attrdict(id=id)
+        cq = { 'id': id,
+               'type': [],
+               '/common/document/content': None,
+               '/common/document/source_uri': None,
+               '/type/content/media_type': { 'name':None,
+                                             'optional': True },
+               #'/type/content/text_encoding': { 'name':None },
+               '/type/content/blob_id':None,
+             }
+        cd = fb.mss.mqlread(cq)
+        if '/type/content' in cd.type:
+            c.media_type = cd['/type/content/media_type'].name
+            #c.text_encoding = cd['/type/content/text_encoding'].name
+            c.sha256 = cd['/type/content/blob_id']
+            return c
+
+        if content_only:
+            raise CmdException('%s is not a content id' % id)
+
+        cid = cd['/common/document/content']
+        if cid is not None:
+            return read_content(cid, content_only=True)
+
+        # in this case we don't have a content object
+        if cd['/common/document/source_uri'] is not None:
+            return None
+
+        raise CmdException('%s is not a content or document id' % id)
+
+
+    content = read_content(id)
+    log.debug('COBJ %r' % content)
+    if content is not None:
+        fileext = media_type_to_extension.get(content.media_type, None)
+    else:
+        fileext = None
+
+    if localfile == '-':
+        ofp = sys.stdout
+    else:
+        if localfile is None:
+            implicit_outfile = True
+            localfile = file
+        elif re.match(r'[/\\]$', localfile):
+            implicit_outfile = True
+            localfile = localfile + file
+        else:
+            implicit_outfile = False
+        localfile = os.path.abspath(localfile)
+
+        # add file extension based on content-type:
+        #  should be an option to disable this
+        if implicit_outfile and fileext is not None:
+            localfile += '.' + fileext
+
+        # if we didn't explicitly name the output file,
+        #  don't destroy an existing file
+        localfile_base = localfile
+        count = 0
+        while implicit_outfile and os.path.exists(localfile):
+            count += 1
+            localfile = '%s.%d' % (localfile_base, count)
+        ofp = open(localfile, 'wb')
+
+    body = fb.mss.trans(id)
+
+    if include_headers:
+        # XXX show content-type, what else?
+        pass
+
+    ofp.write(body)
+
+    if localfile != '-':
+        print ('%s saved (%d bytes)' % (localfile, len(body)))
+        ofp.close()
+
+
+def cmd_put(fb, localfile, id=None, content_type=None):
+    """upload a document to freebase   -- EXPERIMENTAL
+    %prog put localfile [id] [content-type]
+
+    upload the document or image in localfile to given freebase
+    id.  if localfile is '-' the data will be read from stdin.
+
+    if id is missing or empty, a new document will be created.
+    later the id might default to something computed from localfile
+    and any svn attributes it has.
+
+    output: a single line, the id of the document.
+    """
+    if content_type is None:
+        ext = re.sub('^.*\.([^/.]+)$', r'\1', localfile)
+        media_type = extension_to_media_type.get(ext, None)
+
+        if media_type is None:
+            raise CmdException('could not infer a media type from extension %r: please specify it'
+                               % ext)
+
+        if media_type.startswith('text/'):
+            # this is a bad assumption.  should sniff it?
+            text_encoding = 'utf-8'
+            content_type = '%s;charset=%s' % (media_type, text_encoding)
+        else:
+            content_type = media_type
+
+    new_id = None
+    if id is not None:
+        idinfo = fb.mss.mqlread({ 'id': id, 'type': '/common/document' })
+        if idinfo is None:
+            new_id = id
+            id = None
+
+    body = open(localfile, 'rb').read()
+    r = fb.mss.upload(body, content_type, document_id=id)
+
+    if new_id is None:
+        print r.document
+    else:
+        cmd_ln(fb, r.document, new_id)
+        print new_id
+
+
+def cmd_dump(fb, id):
+    """show all properties of a freebase object
+    %prog dump object_id
+    """
+    id = fb.absid(id)
+
+    import inspect
+
+    r = inspect.inspect_object(fb.mss, id)
+    if r is None:
+        raise CmdException('no match for id %r' % id)
+
+    for k in sorted(r.keys()):
+        vs = r[k]
+        for v in vs:
+            id = v.get('id', '')
+
+            name = '%r' % (v.get('name') or v.get('value'))
+            if name == 'None': name = ''
+
+            type = v.get('type', '')
+            if type == '/type/text':
+                extra = v.get('lang', '')
+            elif type == '/type/key':
+                extra = v.get('namespace', '')
+            else:
+                extra = ''
+
+            fb.trow(k, id, name, type, extra)
+                    
+
+def cmd_pget(fb, id, propid):
+    """get a property of a freebase object  -- EXPERIMENTAL
+    %prog pget object_id property_id
+
+    get the property named by property_id from the object.
+
+    XXX output quoting is not well specified.
+
+    property_id must be a fully qualified id for now.
+
+    prints one line for each match.
+
+    if propid ends in '*' this does a wildcard for a particular type.
+    """
+    id = fb.absid(id)
+    proptype, propkey = dirsplit(propid)
+
+    if propkey != '*':
+        # look up the prop
+        q = { 'id': id,
+               propid: [{}],
+             }
+        r = fb.mss.mqlread(q)
+        for v in r[propid]:
+            if 'value' in v:
+                print v.value
+            else:
+                print v.id
+
+    else:
+        # look up the prop
+        q = { 'id': id,
+               '*': [{}],
+             }
+        if isinstance(proptype, basestring):
+            q['type'] = proptype
+
+        r = fb.mss.mqlread(q)
+
+        for k in sorted(r.keys()):
+            v = r[k];
+            if 'value' in v:
+                print '%s %s' % (k, v.value)
+            else:
+                print '%s %s' % (k, v.id)
+
+def cmd_pdel(fb, id, propid, oldval):
+    """delete a property of a freebase object   -- EXPERIMENTAL
+    %prog pdel object_id property_id oldvalue
+
+    set the property named by property_id on the object.
+    value is an id or a json value.  XXX this is ambiguous.
+
+    property_id must be a fully qualified id for now.
+
+    for now you need to provide a "oldval" argument,
+    later this tool will query and perhaps prompt if the
+    deletion is ambiguous.
+
+    prints a single line, either 'deleted' or 'missing'
+    """
+    return cmd_pset(fb, id, propid, None, oldval)
+
+
+def cmd_touch(fb):
+    """bypass any cached query results the service may have.  use sparingly.
+    """
+    fb.mss.mqlflush()
+
+
+def cmd_pset(fb, id, propkey, val, oldval=None, extra=None):
+    """set a property of a freebase object  -- EXPERIMENTAL
+    %prog pset object_id property_id value
+
+    set the property named by property_id on the object.
+    value is an id or a json value.  XXX this is ambiguous.
+
+    property_id must be a fully qualified id for now.
+
+    if the property should be a unique property, this will
+    write with 'connect:update'.  if the property may have
+    multiple, it is written with 'connect:insert'.
+
+    prints a single line, either 'inserted' or 'present'
+    """
+    id = fb.absid(id)
+
+    propid = fb.absprop(propkey)
+
+    # look up the prop
+    pq = { 'id': propid,
+           'type': '/type/property',
+           'name': None,
+           'unique': None,
+           'expected_type': {
+               'id':   None,
+               'name': None,
+               'default_property': None,
+               'optional': True,
+           },
+         }
+
+    prop = fb.mss.mqlread(pq)
+
+    if prop is None:
+        raise CmdException('can\'t resolve property key %r - use an absolute id' % propid);
+
+    if propid.startswith('/type/object/') or propid.startswith('/type/value/'):
+        propkey = re.sub('/type/[^/]+/', '', propid);
+    else:
+        propkey = propid
+
+    wq = { 'id': id,
+           propkey: {
+           }
+         }
+
+    if val is None:
+        val = oldval
+        wq[propkey]['connect'] = 'delete'
+    elif prop.unique:
+        wq[propkey]['connect'] = 'update'
+    else:
+        wq[propkey]['connect'] = 'insert'
+
+    if prop.expected_type is None:
+        wq[propkey]['id'] = val
+    elif prop.expected_type.id not in value_types:
+        wq[propkey]['id'] = val
+    else:
+        wq[propkey]['value'] = val
+    
+        if prop.expected_type.id == '/type/text':
+            if extra is not None:
+                lang = extra
+            else:
+                lang = '/lang/en'
+            wq[propkey]['lang'] = lang
+    
+        if prop.expected_type.id == '/type/key':
+            if extra is not None:
+                wq[propkey]['namespace'] = extra
+            else:
+                raise CmdException('must specify a namespace to pset /type/key')
+
+    r = fb.mss.mqlwrite(wq)
+    print r[propkey]['connect']
+    
+def cmd_login(fb, username=None, password=None):
+    """login to the freebase service
+    %prog login [username [password]]
+
+    cookies are maintained in a file so
+    they are available to the next invocation.
+    prompts for username and password if not given
+    """
+    import getpass
+    if username is None:
+        sys.stdout.write('freebase.com username: ')
+        username = sys.stdin.readline()
+        if not username:
+            raise CmdException('usernmae required for login')
+        username = re.compile('\n$').sub('', username)
+    if password is None:
+        password = getpass.getpass('freebase.com password: ')
+
+    fb.mss.username = username
+    fb.mss.password = password
+    fb.mss.login()
+
+def cmd_logout(fb):
+    """logout from the freebase service
+    %prog logout
+
+    deletes the login cookies
+    """
+    fb.cookiejar.clear(domain=fb.service_host.split(':')[0])
+
+
+def cmd_find(fb, qstr):
+    """print all ids matching a given constraint.
+
+    if the query string starts with "{" it is treated as json.
+    otherwise it is treated as o-rison.
+
+    %prog find
+    """
+    if qstr.startswith('{'):
+        q = simplejson.loads(qstr)
+    else:
+        q = rison.loads('(' + qstr + ')')
+
+    if 'id' not in q:
+        q['id'] = None
+
+    results = fb.mss.mqlreaditer(q)
+    for r in results:
+        print r.id
+
+def cmd_q(fb, qstr):
+    """run a freebase query.
+
+    if the query string starts with "{" it is treated as json.
+    otherwise it is treated as o-rison.
+
+    dump the result as json.
+    
+    %prog q
+    """
+    if qstr.startswith('{'):
+        q = simplejson.loads(qstr)
+    else:
+        q = rison.loads('(' + qstr + ')')
+
+    # results could be streamed with a little more work
+    results = fb.mss.mqlreaditer(q)
+    print simplejson.dumps(list(results), indent=2)
+
+def cmd_open(fb, id):
+    """open a web browser on the given id.  works on OSX only for now.
+    
+    %prog open /some/id
+    """
+    os.system("open 'http://www.freebase.com/view%s'" % id)
+
+
+def cmd_log(fb, id):
+    """log changes pertaining to a given id.
+
+    INCOMPLETE
+    
+    %prog log /some/id
+    """
+
+    null = None
+    true = True
+    false = False
+
+    baseq = {
+            'type': '/type/link',
+            'source': null,
+            'master_property': null,
+            'attribution': null,
+            'timestamp': null,
+            'operation': null,
+            'valid': null,
+            'sort': '-timestamp'
+        };
+    
+    queries = [
+        {
+        'target_value': { '*': null },
+        'target': { 'id': null, 'name': null, 'optional': true },
+        },
+        {
+        'target': { 'id': null, 'name': null },
+        }]
+
+    for i,q in list(enumerate(queries)):
+        q.update(baseq)
+        queries[i] = [q]
+
+
+    valuesfrom,linksfrom = fb.mss.mqlreadmulti(queries)
+
+    for link in linksfrom:
+        # fb.trow(link.master_property.id, ...)
+        print simplejson.dumps(link, indent=2)
+
+    for link in valuesfrom:
+        # fb.trow(link.master_property.id, ...)
+        print simplejson.dumps(link, indent=2)
+
+    
diff --git a/freebase/fcl/fbutil.py b/freebase/fcl/fbutil.py
new file mode 100755
index 0000000..ed1e511
--- /dev/null
+++ b/freebase/fcl/fbutil.py
@@ -0,0 +1,113 @@
+# ==================================================================
+# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ====================================================================
+
+
+import re
+
+import logging
+log = logging.getLogger()
+
+
+class FbException(Exception):
+    pass
+
+class CmdException(Exception):
+    pass
+
+media_types = {
+            'html': ['text/html'],
+            'txt':  ['text/plain'],
+
+            'xml':  ['text/xml',
+                     'application/xml'],
+
+            'atom': ['application/atom+xml'],
+
+            'js':   ['text/javascript',
+                     'application/javascript',
+                     'application/x-javascript'],
+
+            'json': ['application/json'],
+
+            'jpg':  ['image/jpeg',
+                     'image/pjpeg'],
+
+            'gif':  ['image/gif'],
+
+            'png':  ['image/png'],
+    }
+
+extension_to_media_type = dict([(k,vs[0]) for k,vs in media_types.items()])
+media_type_to_extension = {}
+for k,vs in media_types.items():
+    for v in vs:
+        media_type_to_extension[v] = k
+
+
+
+
+DIRSPLIT = re.compile(r'^(.+)/([^/]+)$')
+
+def dirsplit_unsafe(id):
+    m = DIRSPLIT.match(id)
+    if m is None:
+        return (None, id)
+    dir,file = m.groups()
+    return (dir,file)
+
+def dirsplit(id):
+    dir,file = dirsplit_unsafe(id)
+    if dir == '/guid':
+        raise FbException('%r is not a freebase keypath' % (id,))
+    return (dir,file)
+
+value_types = [
+    '/type/text',
+    '/type/key',
+    '/type/rawstring',
+    '/type/float',
+    '/type/int',
+    '/type/boolean',
+    '/type/uri',
+    '/type/datetime',
+    '/type/id',
+    '/type/enumeration',
+    ]
+
+default_propkeys = {
+    'value': '/type/value/value',
+    'id': '/type/object/id',
+    'guid': '/type/object/guid',
+    'type': '/type/object/type',
+    'name': '/type/object/name',
+    'key': '/type/object/key',
+    'timestamp': '/type/object/timestamp',
+    'permission': '/type/object/permission',
+    'creator': '/type/object/creator',
+    'attribution': '/type/object/attribution'
+};
diff --git a/freebase/fcl/fcl.py b/freebase/fcl/fcl.py
new file mode 100755
index 0000000..a0299b8
--- /dev/null
+++ b/freebase/fcl/fcl.py
@@ -0,0 +1,276 @@
+#!/usr/bin/env python
+# ==================================================================
+# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ====================================================================
+
+import os, sys, re, time, stat
+
+from optparse import OptionParser
+import getpass
+import cookielib
+import logging
+import simplejson
+
+
+from fbutil import FbException, CmdException, log, default_propkeys
+console = logging.StreamHandler()
+log.addHandler(console)
+
+from freebase.api import HTTPMetawebSession, MetawebError, attrdict
+
+_cookiedir = None
+if os.environ.has_key('HOME'):
+    _cookiedir = os.path.join(os.environ['HOME'], '.pyfreebase')
+
+
+class Command(object):
+    def __init__(self, module, name, func):
+        self.module = module
+        self.name = name
+        self.func = func
+
+        PROG = re.compile(r'\%prog')
+        NEWLINE = re.compile(r'(?:\r?\n)+')
+
+        # fill in self.doc
+        if isinstance(func.__doc__, basestring):
+            doc = PROG.sub('fcl', func.__doc__ + '\n')
+            self.shortdoc, self.doc = NEWLINE.split(doc, 1)
+        else:
+            self.shortdoc = '(missing documentation)'
+            self.doc = '(missing documentation)'
+
+
+
+class FbCommandHandler(object):
+
+    def __init__(self):
+        self.service_host = 'www.freebase.com'
+        self.cookiejar = None
+        self.cwid = ''
+        self.progpath = 'fcl'
+        self.commands = {}
+
+        self.cookiefile = None
+        if _cookiedir is not None:
+            self.cookiefile = os.path.join(_cookiedir, 'cookiejar')
+
+
+    def init(self):
+        if self.cookiefile is not None:
+            self.cookiejar = cookielib.LWPCookieJar(self.cookiefile)
+            if os.path.exists(self.cookiefile):
+                try:
+                    self.cookiejar.load(ignore_discard=True)
+                except cookielib.LoadError:
+                    log.warn('error loading cookies')
+
+        #print 'start cookies %r' % self.cookiejar
+
+        self.mss = HTTPMetawebSession(self.service_host,
+                                      cookiejar=self.cookiejar)
+
+
+    def absid(self, path):
+        if path is None:
+            path = ''
+        if path.startswith('/'):
+            return path
+
+        if not isinstance(self.cwid, basestring) or not self.cwid.startswith('/'):
+            # svn cwid support is disabled because it relies on some svn glue that
+            # was not sufficiently well thought out.
+            # raise CmdException("can't resolve relative id %r without cwid - see 'fcl help pwid'" % (path))
+            raise CmdException("no support for relative id %r" % (path))
+
+        if path == '' or path == '.':
+            return self.cwid
+
+        return self.cwid + '/' + path
+
+
+    def absprop(self, propkey):
+        if propkey.startswith('/'):
+            return propkey
+
+        # check schemas of /type/object and /type/value,
+        #  as well as other reserved names
+        if propkey in default_propkeys:
+            return default_propkeys[propkey]
+
+        return self.absid(propkey)
+
+    def thead(self, *args):
+        strs = ['%r' % arg
+                for arg in args]
+        print '\t'.join(strs)
+
+    def trow(self, *args):
+        print '\t'.join(args)
+        return
+        strs = ['%r' % arg
+                for arg in args]
+        print '\t'.join(strs)
+
+    def save(self):
+        #print 'end cookies %r' % self.cookiejar
+        if _cookiedir and self.cookiefile.startswith(_cookiedir):
+            # create private cookiedir if needed
+            if not os.path.exists(_cookiedir):
+                os.mkdir(_cookiedir, 0700)
+                os.chmod(_cookiedir, stat.S_IRWXU)
+
+        if self.cookiejar is None:
+            return
+
+        self.cookiejar.save(ignore_discard=True)
+
+        # save the cwd and other state too
+
+
+    def import_commands(self, modname):
+        """
+        import new fb commands from a file
+        """
+        namespace = {}
+    
+        pyimport = 'from %s import *' % modname
+        exec pyimport in namespace
+        mod = sys.modules.get(modname)
+
+        commands = [Command(mod, k[4:], getattr(mod, k))
+                    for k in getattr(mod, '__all__', dir(mod))
+                    if (k.startswith('cmd_')
+                        and callable(getattr(mod, k)))]
+
+        
+        for cmd in commands:
+            log.info('importing %r' % ((cmd.name, cmd.func),))
+            self.commands[cmd.name] = cmd
+    
+        log.info('imported %r from %r' % (modname, mod.__file__))
+
+
+    def dispatch(self, cmd, args):
+        if cmd in self.commands:
+            try:
+                self.commands[cmd].func(self, *args)
+            except KeyboardInterrupt, e:
+                sys.stderr.write('%s\n' % (str(e),))
+            except FbException, e:
+                sys.stderr.write('%s\n' % (str(e),))
+            except CmdException, e:
+                sys.stderr.write('%s\n' % (str(e),))
+            except MetawebError, e:
+                sys.stderr.write('%s\n' % (str(e),))
+        else:
+            self.oparser.error('unknown subcommand %r, try "%s help"' % (cmd, self.progpath))
+
+        self.save()
+        
+
+    def cmdline_main(self):
+        op = OptionParser(usage='%prog [options] command [args...] ')
+        self.oparser = op
+
+        op.disable_interspersed_args()
+
+        op.add_option('-d', '--debug', dest='debug',
+                        default=False, action='store_true',
+                        help='turn on debugging output')
+
+        op.add_option('-v', '--verbose', dest='verbose',
+                        default=False, action='store_true',
+                        help='verbose output')
+
+        op.add_option('-V', '--very-verbose', dest='very_verbose',
+                        default=False, action='store_true',
+                        help='lots of debug output')
+
+        op.add_option('-s', '--service', dest='service_host',
+                        metavar='HOST',
+                        default=self.service_host,
+                        help='Freebase HTTP service address:port')
+
+        op.add_option('-S', '--sandbox', dest='use_sandbox',
+                        default=False, action='store_true',
+                        help='shortcut for --service=sandbox.freebase.com')
+
+        op.add_option('-c', '--cookiejar', dest='cookiefile',
+                        metavar='FILE',
+                        default=self.cookiefile,
+                        help='Cookie storage file (will be created if missing)')
+        options,args = op.parse_args()
+
+        if len(args) < 1:
+            op.error('required subcommand missing')
+
+
+        loglevel = logging.WARNING
+        if options.verbose:
+            loglevel = logging.INFO
+        if options.very_verbose:
+            loglevel = logging.DEBUG
+
+        console.setLevel(loglevel)
+        log.setLevel(loglevel)
+
+        if options.use_sandbox:
+            self.service_host = 'sandbox.freebase.com'
+        else:
+            self.service_host = options.service_host
+
+        self.cookiefile = options.cookiefile
+        #self.progpath = sys.argv[0]
+        
+        self.init()
+
+        self.mss.log.setLevel(loglevel)
+        self.mss.log.addHandler(console)
+
+        self.import_commands('freebase.fcl.commands')
+        self.import_commands('freebase.fcl.mktype')
+
+        cmd = args.pop(0)
+        self.dispatch(cmd, args)
+
+# entry point for script
+def main():
+    try:
+        # turn off crlf output on windows so we work properly
+        # with unix tools.
+        import msvcrt
+        msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+        msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
+    except ImportError:
+        pass
+
+    fb = FbCommandHandler()
+    fb.cmdline_main()
+
+if __name__ == '__main__':
+    main()
diff --git a/freebase/fcl/inspect.py b/freebase/fcl/inspect.py
new file mode 100755
index 0000000..8cd04d2
--- /dev/null
+++ b/freebase/fcl/inspect.py
@@ -0,0 +1,218 @@
+# ==================================================================
+# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ====================================================================
+
+
+#
+#
+#  wrap all the nastiness needed for a general mql inspect query
+#
+#
+
+import os, sys, re
+
+
+null = None
+true = True
+false = False
+
+inspect_query = {
+          'name': null,
+          'type': [],
+
+          '/type/reflect/any_master': [{
+            'optional':true,
+            'id': null,
+            'name': null,
+            'link': {
+              'master_property': {
+                'id': null,
+                'schema': null
+              }
+            }
+          }],
+
+          '/type/reflect/any_reverse': [{
+            'optional':true,
+            'id': null,
+            'name': null,
+            'link': {
+              'master_property': {
+                'id':null,
+                'schema': null,
+                'expected_type': null,
+                'reverse_property': {
+                  'id': null,
+                  'schema': null,
+                  'optional': true
+                }
+              }
+            }
+          }],
+
+          '/type/reflect/any_value': [{
+            'optional':true,
+            'value': null,
+            'link': {
+              'master_property': {
+                'id':null,
+                'schema': null,
+                'expected_type': null
+              },
+            }
+          }],
+
+          't:/type/reflect/any_value': [{
+            'optional':true,
+            'type': '/type/text',
+            'value': null,
+            'lang': null,
+            'link': {
+              'master_property': {
+                'id':null,
+                'schema': null
+              },
+            }
+          }],
+          
+          '/type/object/creator': [{
+            'optional':true,
+            'id':null,
+            'name':null
+          }],
+          '/type/object/timestamp': [{
+            'optional':true,
+            'value': null,
+          }],
+
+          '/type/object/key': [{
+            'optional':true,
+            'value': null,
+            'namespace': null
+          }],
+          '/type/namespace/keys': [{
+            'optional':true,
+            'value': null,
+            'namespace': null
+          }]
+}
+
+
+def transform_result(result):
+    proptypes = {}
+    props = {}
+
+    # copy a property from a /type/reflect clause
+    def pushtype(propdesc, prop):
+        tid = propdesc['schema']
+        propid = propdesc['id']
+
+        if isinstance(prop, dict):
+            prop = dict(prop)
+            if 'link' in prop:
+                prop.pop('link')
+
+        if tid not in proptypes:
+            proptypes[tid] = {}
+        if propid not in proptypes[tid]:
+            proptypes[tid][propid] = []
+
+        if propid not in props:
+            props[propid] = []
+        props[propid].append(prop)
+    
+    # copy a property that isn't enumerated by /type/reflect
+    def pushprop(propid):
+        ps = result[propid]
+        if ps is None:
+            return
+
+        # hack to infer the schema from id, not always reliable!
+        schema = re.sub(r'/[^/]+$', '', propid)
+        keyprop = dict(id=propid, schema=schema)
+        for p in ps:
+            pushtype(keyprop, p)
+
+    ps = result['/type/reflect/any_master'] or []
+    for p in ps:
+        propdesc = p.link.master_property
+        pushtype(propdesc, p)
+
+    # non-text non-key values
+    ps = result['/type/reflect/any_value'] or []
+    for p in ps:
+        propdesc = p.link.master_property
+
+        # /type/text values are queried specially
+        #  so that we can get the lang, so ignore
+        #  them here.
+        if propdesc.expected_type == '/type/text':
+            continue
+
+        pushtype(propdesc, p)
+
+    # text values
+    ps = result['t:/type/reflect/any_value'] or []
+    for p in ps:
+        propdesc = p.link.master_property
+        pushtype(propdesc, p)
+
+    pushprop('/type/object/creator')
+    pushprop('/type/object/timestamp')
+    pushprop('/type/object/key')
+    pushprop('/type/namespace/keys')
+
+    # now the reverse properties
+    ps = result['/type/reflect/any_reverse'] or []
+    for prop in ps:
+        propdesc = prop.link.master_property.reverse_property
+
+        # synthetic property descriptor for the reverse of
+        #  a property with no reverse descriptor.
+        # note the bogus id starting with '-'.
+        if propdesc is None:
+            #  schema = prop.link.master_property.expected_type
+            #  if schema is None:
+            #      schema = 'other'
+
+            schema = 'other'
+            propdesc = dict(id='-' + prop.link.master_property.id,
+                            schema=schema)
+
+        pushtype(propdesc, prop)
+
+    #return proptypes
+    return props
+
+
+def inspect_object(mss, id):
+    q = dict(inspect_query)
+    q['id'] = id
+    r = mss.mqlread(q)
+    if r is None:
+        return None
+    return transform_result(r)
diff --git a/freebase/fcl/mktype.py b/freebase/fcl/mktype.py
new file mode 100755
index 0000000..30f8df7
--- /dev/null
+++ b/freebase/fcl/mktype.py
@@ -0,0 +1,179 @@
+# ==================================================================
+# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ====================================================================
+
+import os, sys, re, time
+from fbutil import *
+
+
+def cmd_mkobj(fb, id, typeid='/common/topic', name=''):
+    """create a new object with a given type  -- EXPERIMENTAL
+    %prog mkobj new_id typeid name
+
+    create a new object with type typeid at the given
+    namespace location.
+
+    if present, name gives the display name of the new object.
+    
+    """
+    id = fb.absid(id)
+    nsid, key = dirsplit(id)
+
+    typeid = fb.absid(typeid)
+
+    if name == '':
+        name = key
+
+    wq = { 'create': 'unless_exists',
+           'id': None,
+           'name': name,
+           'type': typeid,
+           'key':{
+               'namespace': nsid,
+               'value': key
+           },
+         }
+
+    # TODO add included types
+
+    r = fb.mss.mqlwrite(wq)
+    print r.id,r.create
+
+def cmd_mktype(fb, id, name=''):
+    """create a new type  -- EXPERIMENTAL
+    %prog mktype new_id name
+
+    create a new object with type Type at the given
+    namespace location.
+
+    this doesn't create any type hints.
+
+    if present, name gives the display name of the new property
+    """
+    id = fb.absid(id)
+
+    nsid, key = dirsplit(id)
+
+    if name == '':
+        name = key
+
+    wq = { 'create': 'unless_exists',
+           'id': None,
+           'name': name,
+           'type': '/type/type',
+           'key':{
+               'namespace': nsid,
+               'value': key
+           },
+         }
+
+    r = fb.mss.mqlwrite(wq)
+    print r.id,r.create
+
+def mkprop(fb, typeid, key, name='', vtype=None, master_property=None):
+    """helper to create a new property
+    """
+    if name == '':
+        name = key
+
+    wq = { 'create': 'unless_exists',
+           'id': None,
+           'type': '/type/property',
+           'name': name,
+           'schema': typeid,
+           'key': {
+               'namespace': typeid,
+               'value': key
+            }
+         }
+
+    if vtype is not None:
+        wq['expected_type'] = vtype
+    if master_property is not None:
+        wq['master_property'] = master_property
+
+    return fb.mss.mqlwrite(wq)
+
+
+def cmd_mkprop(fb, id, name='', vtype=None, revkey=None, revname=''):
+    """create a new property  -- EXPERIMENTAL
+    %prog mkprop new_id [name] [expected_type] [reverse_property] [reverse_name]
+
+    create a new object with type Property at the given
+    location.  creates both the "schema" and "key" links
+    for the property, but doesn't create any freebase property
+    hints.
+
+    if present, name gives the display name of the new property
+    """
+    id = fb.absid(id)
+    if vtype is not None:
+        vtype = fb.absid(vtype)
+
+    typeid, key = dirsplit(id)
+
+
+    r = mkprop(fb, typeid, key, name, vtype)
+
+    # write the reverse property if specified
+
+    print r.id, r.create
+
+    if revkey is None:
+        return
+
+    assert vtype is not None
+
+    rr = mkprop(fb, vtype, revkey, revname, typeid, id)
+    print rr.id, rr.create
+
+
+def cmd_publish_type(fb, typeid):
+    """try to publish a freebase type for the client
+    %prog publish_type typeid
+
+    set /freebase/type_profile/published to the /freebase/type_status
+    instance named 'Published'
+  
+
+    should also try to set the domain to some namespace that
+    has type:/type/domain
+
+    """
+    id = fb.absid(typeid)
+
+    w = {
+        'id': id,
+        '/freebase/type_profile/published': {
+            'connect': 'insert',
+            'type': '/freebase/type_status',
+            'name': 'Published'
+        }
+    }
+    r = fb.mss.mqlwrite(w)
+
+    print r['/freebase/type_profile/published']['connect']
diff --git a/freebase/rison.py b/freebase/rison.py
new file mode 100755
index 0000000..2f4eebc
--- /dev/null
+++ b/freebase/rison.py
@@ -0,0 +1,308 @@
+#
+#  rison for python (parser only so far)
+#    see http://mjtemplate.org/examples/rison.html for more info
+#
+
+######################################################################
+#
+# the rison parser is based on javascript openlaszlo-json:
+#    Author: Oliver Steele
+#    Copyright: Copyright 2006 Oliver Steele.  All rights reserved.
+#    Homepage: http:#osteele.com/sources/openlaszlo/json
+#    License: MIT License.
+#    Version: 1.0
+#
+
+# hacked by nix for use in uris
+# ported to python by nix
+#
+#  TODO
+#
+#  switch to unicode
+#  fall through to simplejson if first char is not in '!(' -
+#   this allows code to use just one parser
+#
+
+
+import os, sys, re
+#import simplejson
+simplejson = None
+
+class ParserException(Exception):
+    pass
+
+class Parser(object):
+    WHITESPACE = ''
+    #WHITESPACE = " \t\n\r\f"
+
+    # we divide the uri-safe glyphs into three sets
+    # <rison> and <reserved> classes are illegal in ids.
+    #   <rison> - used by rison (possibly later)
+    #   <reserved> - not common in strings, reserved
+    #not_idchar  = "'!=:(),*@$;&";
+
+    idchar_punctuation = '_-./~'
+    not_idchar  = ''.join([c for c in (chr(i) for i in range(127))
+                           if not (c.isalnum()
+                                   or c in idchar_punctuation)])
+
+    # additionally, we need to distinguish ids and numbers by first char
+    not_idstart = "-0123456789";
+
+    # regexp string matching a valid id
+    idrx = ('[^' + not_idstart + not_idchar + 
+            '][^' + not_idchar + ']*')
+
+    # regexp to check for valid rison ids
+    id_ok_re = re.compile('^' + idrx + '$', re.M)
+
+    # regexp to find the end of an id when parsing
+    next_id_re = re.compile(idrx, re.M)
+
+    def parse_json(self, str):
+        if len(str) > 0 and str[0] not in '!(':
+            return simplejson.loads(str)
+        return self.parse(str)
+
+    def parse(self, str):
+        self.string = str
+        self.index = 0
+
+        value = self.readValue()
+        if self.next():
+            raise ParserException("unable to parse rison string %r" % (str,))
+        return value
+    
+    def readValue(self):
+        c = self.next()
+
+        if c == '!':
+            return self.parse_bang()
+        if c == '(':
+            return self.parse_open_paren()
+        if c == "'":
+            return self.parse_single_quote()
+        if c in '-0123456789':
+            return self.parse_number()
+
+        # fell through table, parse as an id
+        s = self.string
+        i = self.index-1
+
+        m = self.next_id_re.match(s, i)
+        if m:
+            id = m.group(0)
+            self.index = i + len(id)
+            return id  # a string
+
+        if c:
+            raise ParserException("invalid character: '" + c + "'")
+        raise ParserException("empty expression")
+
+    def parse_array(self):
+        ar = []
+        while 1:
+            c = self.next()
+            if c == ')':
+                return ar
+
+            if c is None:
+                raise ParserException("unmatched '!('")
+
+            if len(ar):
+                if c != ',':
+                    raise ParserException("missing ','")
+            elif c == ',':
+                raise ParserException("extra ','")
+            else:
+                self.index -= 1
+            n = self.readValue()
+            ar.append(n)
+
+        return ar
+
+    def parse_bang (self):
+        s = self.string
+        c = s[self.index]
+        self.index += 1
+        if c is None:
+            raise ParserException('"!" at end of input')
+        if c not in self.bangs:
+            raise ParserException('unknown literal: "!' + c + '"')
+        x = self.bangs[c]
+        if callable(x):
+            return x(self)
+
+        return x
+
+
+    def parse_open_paren (self):
+        count = 0
+        o = {}
+
+        while 1:
+            c = self.next()
+            if c == ')':
+                return o
+            if count:
+                if c != ',':
+                    raise ParserException("missing ','")
+            elif c == ',':
+                raise ParserException("extra ','")
+            else:
+                self.index -= 1
+            k = self.readValue()
+
+            if self.next() != ':':
+                raise ParserException("missing ':'")
+            v = self.readValue()
+
+            o[k] = v
+            count += 1
+        
+
+    def parse_single_quote(self):
+        s = self.string
+        i = self.index
+        start = i
+        segments = []
+
+        while 1:
+            if i >= len(s):
+                raise ParserException('unmatched "\'"')
+
+            c = s[i]
+            i += 1
+            if c == "'":
+                break
+
+            if c == '!':
+                if start < i-1:
+                    segments.append(s[start:i-1])
+                c = s[i]
+                i += 1
+                if c in "!'":
+                    segments.append(c)
+                else:
+                    raise ParserException('invalid string escape: "!'+c+'"')
+                
+                start = i
+            
+        
+        if start < i-1:
+            segments.append(s[start:i-1])
+        self.index = i
+        return ''.join(segments)
+
+
+    # Also any number start (digit or '-')
+    def parse_number(self):
+        s = self.string
+        i = self.index
+        start = i-1
+        state = 'int'
+        permittedSigns = '-'
+        transitions = {
+            'int+.': 'frac',
+            'int+e': 'exp',
+            'frac+e': 'exp'
+            }
+        
+        while 1:
+            if i >= len(s):
+                i += 1
+                break
+
+            c = s[i]
+            i += 1
+
+            if '0' <= c and c <= '9':
+                continue
+
+            if permittedSigns.find(c) >= 0:
+                permittedSigns = ''
+                continue
+
+            state = transitions.get(state + '+' + c.lower(), None)
+            if state is None:
+                break
+            if state == 'exp':
+                permittedSigns = '-'
+
+        self.index = i - 1
+        s = s[start:self.index]
+        if s == '-':
+            raise ParserException("invalid number")
+        if re.search('[.e]', s):
+            return float(s)
+        return int(s)
+    
+    # return the next non-whitespace character, or undefined
+    def next(self):
+        l = len(self.string)
+        s = self.string
+        i = self.index
+
+        while 1:
+            if i == len(s):
+                return None
+            c = s[i]
+            i += 1
+            if c not in self.WHITESPACE:
+                break
+
+        self.index = i
+        return c
+
+
+    bangs = {
+        't': True,
+        'f': False,
+        'n': None,
+        '(': parse_array
+        }
+
+
+def loads(s):
+    return Parser().parse(s)
+
+if __name__ == '__main__':
+    p = Parser()
+
+    rison_examples = [
+        "(a:0,b:1)",
+        "(a:0,b:foo,c:'23skidoo')",
+        "!t",
+        "!f",
+        "!n",
+        "''",
+        "0",
+        "1.5",
+        "-3",
+        "1e30",
+        "1e-30",
+        "G.",
+        "a",
+        "'0a'",
+        "'abc def'",
+        "()",
+        "(a:0)",
+        "(id:!n,type:/common/document)",
+        "!()",
+        "!(!t,!f,!n,'')",
+        "'-h'",
+        "a-z",
+        "'wow!!'",
+        "domain.com",
+        "'user@domain.com'",
+        "'US $10'",
+        "'can!'t'",
+    ];
+
+    for s in rison_examples:
+        print
+        print '*'*70
+        print
+        print s
+        
+        print '%r' % (p.parse(s),)
diff --git a/freebase/sandbox.py b/freebase/sandbox.py
new file mode 100644
index 0000000..2d11be2
--- /dev/null
+++ b/freebase/sandbox.py
@@ -0,0 +1,65 @@
+#!/usr/bin/python
+# ========================================================================
+# Copyright (c) 2007, Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ========================================================================
+
+import sys
+from freebase.api.session import HTTPMetawebSession
+
+base = HTTPMetawebSession("sandbox.freebase.com")
+
+__all__ = ["HTTPMetawebSession"]
+
+# we want to add base's functions to __init__.py
+# so that we can say freebase.func() and really
+# just call base.func()
+
+# a little trick to refer to ourselves
+self = sys.modules[__name__]
+
+for funcname in dir(base):
+    
+    # we only want the 'real' functions
+    if not funcname.startswith("_"):
+        func = getattr(base, funcname)
+        
+        # let's make sure we're getting functions
+        # instead of constants or whatever
+        if callable(func):
+            
+            # we're setting these functions
+            # so that they can be called like
+            # freebase.funcname -> base.func()
+            setattr(self, funcname, func)
+            
+            # make sure we import the base's
+            # functions if we import freebase
+            __all__.append(funcname)
+
+# we don't want any self-referencing
+# business going. Plus, this is cleaner.
+del self
diff --git a/freebase/uritemplate.py b/freebase/uritemplate.py
new file mode 100755
index 0000000..0a81147
--- /dev/null
+++ b/freebase/uritemplate.py
@@ -0,0 +1,143 @@
+# ========================================================================
+# Copyright (c) 2007, Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ========================================================================
+
+#
+# URI Templating in Python
+#
+# see http://bitworking.org/projects/URI-Templates/
+#  and http://bitworking.org/news/URI_Templates
+#
+# note that this implementation may go away soon in
+#  favor of joe gregorio's own code:
+#     http://code.google.com/p/uri-templates/
+#
+#
+# this implementation can also parse URIs, as long as the
+#   template is sufficiently specific.  to allow this to work
+#   the '/' character is forbidden in keys when parsing.  
+#   later it should be possible to loosen this restriction.
+#
+#
+# example:
+#   from whatever.uritemplate import expand_uri_template
+#   expand_uri_template('http://{host}/{file}',
+#                       dict(host='example.org',
+#                            file='fred'))
+#
+# TODO:
+#   allow parsing to be aware of http://www. and trailing /
+#   nail down quoting issues
+#
+
+import os, sys, re
+import urllib
+
+__all__ = ['expand_uri_template', 'URITemplate']
+
+
+def expand_uri_template(template, args):
+    """Expand a URI template using the given args dictionary.
+    """
+    return URITemplate(template).run(args)
+
+def _uri_encode_var(v):
+    return urllib.quote(v, safe="-_.~!$&'()*+,;=:/?[]#@")
+
+
+class URITemplate(object):
+    """a URITemplate is a URI with simple variable substitution.
+    """
+
+    VARREF = re.compile(r'\{([0-9a-zA-Z_]+)\}')
+
+    def __init__(self, s):
+        """Compile a URITemplate from a string.
+        """
+        self.template = s;
+
+        self.params = []
+        tsplit = self.VARREF.split(s)
+        rxs = ['^']
+        for i in range(len(tsplit)):
+            if i % 2:
+                # track the vars used
+                self.params.append(tsplit[i])
+                # vars match any string
+                # vars are assumed to lack '/' - this is imperfect...
+                rxs.append('([^/]*)')
+            else:
+                # quote special chars regexp interpretation
+                rxs.append(re.escape(tsplit[i]))
+        rxs.append('$')
+        self._parser = re.compile(''.join(rxs))
+
+    def __repr__(self):
+        return '<URITemplate %r>' % self.template
+
+    def run (self, args):
+        """Expand the template using the given args.
+        """
+        def repl(m):
+            key = m.group(1)
+            return _uri_encode_var(args.get(key, ''))
+        uri = self.VARREF.sub(repl,self.template)
+
+        #if self.parse(uri) is None:
+        #    print 're-parsing generated uri failed: %r, %r' % (uri, self.template)
+        return uri
+
+
+    def parse(self, uri):
+        """Try to parse a URI, extracting variable values.
+        """
+        m = self._parser.match(uri)
+        if m is None:
+            return None
+        return dict(zip(self.params, m.groups()))
+
+
+if __name__ == '__main__':
+    #
+    #  testcases are imported from the URI::Template module on CPAN
+    #
+    import urllib2, simplejson
+    fp = urllib2.urlopen('http://search.cpan.org/src/BRICAS/URI-Template-0.09/t/data/spec.json')
+    testsuite = simplejson.loads(fp.read())
+    vars = dict([(k.encode('utf-8'),v) for k,v in testsuite['variables'].items()])
+    nsucceed = 0
+    nfail = 0
+    for test in testsuite['tests']:
+        ut = URITemplate(test['template'])
+        uri = ut.run(vars)
+        if uri != test['expected']:
+            print 'FAILED %r expected %r' % (uri, test['expected'])
+            print ' vars: %r' % (vars,)
+            nfail += 1
+        else:
+            nsucceed += 1
+    print 'tests: %d succeeded, %d failed' % (nsucceed, nfail)