Initial import of sources copied from Metaweb internal source control.

git-svn-id: http://freebase-python.googlecode.com/svn/trunk@2 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2

4 files changed, 698 insertions, 0 deletions

diff --git a/trunk/freebase/__init__.py b/trunk/freebase/__init__.py
new file mode 100755
index 0000000..d289e09
--- /dev/null
+++ b/trunk/freebase/__init__.py
@@ -0,0 +1,2 @@
+
+pass
diff --git a/trunk/freebase/api/__init__.py b/trunk/freebase/api/__init__.py
new file mode 100755
index 0000000..959259f
--- /dev/null
+++ b/trunk/freebase/api/__init__.py
@@ -0,0 +1,2 @@
+
+from session import HTTPMetawebSession, MetawebError
diff --git a/trunk/freebase/api/session.py b/trunk/freebase/api/session.py
new file mode 100644
index 0000000..b3ec825
--- /dev/null
+++ b/trunk/freebase/api/session.py
@@ -0,0 +1,391 @@
+# ========================================================================
+# Copyright (c) 2007, Metaweb Technologies, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+# 
+# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
+# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# ========================================================================
+
+#
+#  declarations for external metaweb api.
+#
+#
+#    from metaweb.api import HTTPMetawebSession
+#
+#    mss = HTTPMetawebSession('sandbox.freebase.com')
+#    print mss.mqlread([dict(name=None, type='/type/type')])
+#
+#
+#
+
+import os, sys, re
+import urllib2
+import cookielib
+import simplejson
+from urllib import quote as urlquote
+import pprint
+import socket
+
+import logging
+log = logging.getLogger()
+log.setLevel(logging.DEBUG)
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+# add the handler to the root logger
+log.addHandler(console)
+
+
+__all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession']
+
+__version__ = '0.1'
+
+import simplejson.encoder
+simplejson.JSONEncoder.item_separator = ','
+simplejson.JSONEncoder.key_separator = ':'
+#delattr(simplejson.encoder.ESCAPE_DCT, '/')
+
+def urlencode_weak(s):
+    return urlquote(s, safe='{}[],"/:$')
+
+
+# from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/361668
+class attrdict(dict):
+    """A dict whose items can also be accessed as member variables.
+
+    >>> d = attrdict(a=1, b=2)
+    >>> d['c'] = 3
+    >>> print d.a, d.b, d.c
+    1 2 3
+    >>> d.b = 10
+    >>> print d['b']
+    10
+
+    # but be careful, it's easy to hide methods
+    >>> print d.get('c')
+    3
+    >>> d['get'] = 4
+    >>> print d.get('a')
+    Traceback (most recent call last):
+    TypeError: 'int' object is not callable
+    """
+    def __init__(self, *args, **kwargs):
+        dict.__init__(self, *args, **kwargs)
+        self.__dict__ = self
+
+
+
+# TODO expose the common parts of the result envelope
+class MetawebError(Exception):
+    """
+    an error report from the metaweb service.
+    """
+    pass
+
+
+
+# TODO right now this is a completely unnecessary superclass.
+#  is there enough common behavior between session types
+#  to justify it?
+class MetawebSession(object):
+    """
+    MetawebSession is the base class for MetawebSession, subclassed for
+    different connection types.  Only http is available externally.
+
+    This is more of an interface than 
+    """
+
+    # interface definition here...
+    
+
+class HTTPMetawebSession(MetawebSession):
+    """
+    a MetawebSession is a request/response queue.
+
+    this version uses the HTTP api, and is synchronous.
+    """
+    # share cookies across sessions, so that different sessions can
+    #  see each other's writes immediately.
+    _cookies = cookielib.CookieJar()
+    _cookiespy = urllib2.HTTPCookieProcessor(_cookies)
+
+    def __init__(self, service_url, username=None, password=None, prev_session=None):
+        """
+        create a new MetawebSession for interacting with the Metaweb.
+
+        a new session will inherit state from prev_session if present, 
+        """
+        super(HTTPMetawebSession, self).__init__()
+
+        assert not service_url.endswith('/')
+        if not '/' in service_url:  # plain host:port
+            service_url = 'http://' + service_url
+
+        self.service_url = service_url
+
+        self.username = username
+        self.password = password
+
+        self.tid = None
+
+        if prev_session:
+            self.service_url = prev.service_url
+
+    def _httpreq(self, service_path, method='GET', body=None, form=None,
+                 headers=None, **kws):
+        """
+        make an http request to the service.
+
+        form arguments are encoded in the url, even for POST, if a non-form
+        content-type is given for the body.
+        """
+
+        if method == 'POST':
+            assert body is not None or form is not None
+        elif method == 'GET':
+            assert body is None
+        else:
+            assert 0, 'unknown method %s' % method
+
+        log.debug('HTTPREQ: %s %s %s %s', service_path, method, kws, self._cookies)
+
+        url = self.service_url + service_path
+
+        if headers is None:
+            headers = []
+
+        # XXX This is a lousy way to parse Content-Type, where is
+        #  the library?
+        ct = None
+        for k,v in headers:
+            if k.lower() == 'content-type':
+                ct = v.split(';')[0]
+                break
+
+        if body is not None:
+            # if body is provided, content-type had better be too
+            assert ct is not None
+
+        if form is not None:
+            qstr = '&'.join(['%s=%s' % (urlencode_weak(k), urlencode_weak(v))
+                             for k,v in form.items()])
+            if method == 'POST':
+                # put the args on the url if we're putting something else
+                # in the body.  this is used to add args to raw uploads.
+                if body is not None:
+                    url += '?' + qstr
+                else:
+                    if ct is None:
+                        # XXX encoding and stuff
+                        ct = 'application/x-www-form-urlencoded'
+                        headers.append(('content-type', ct))
+
+                    if ct == 'multipart/form-encoded':
+                        # XXX fixme
+                        raise NotImplementedError
+                    elif ct == 'application/x-www-form-urlencoded':
+                        body = qstr
+            else:
+                # for all methods other than POST, use the url
+                url += '?' + qstr
+
+        req = urllib2.Request(url, body, dict(headers), **kws)
+
+        self.opener = urllib2.build_opener(self._cookiespy)
+
+        # assure the service that this isn't a CSRF form submission
+        self.opener.addheaders = [('User-agent',
+                                   'python freebase.api-%s' % __version__),
+                                  ('X-Metaweb-Request', 'Python')]
+
+        if self.tid is not None:
+            self.opener.addheaders.append(('X-Metaweb-TID', self.tid))
+
+        try:
+            if headers:
+                self.opener.addheaders += headers
+            resp = self.opener.open(req)
+
+        except socket.error, e:
+            log.error('SOCKET FAILURE: %s', e.fp.read())
+            raise MetawebError, 'failed contacting %s: %s' % (url, str(e))
+
+        except urllib2.HTTPError, e:
+            if e.code == 400 and e.info().type == 'application/json':
+                r = self._readjson(e.fp)
+                for msg in r.messages:
+                    if msg.get('level', 'error') == 'error':
+                        raise MetawebError(u'%s %s %r' % (msg.get('code',''), msg.message, msg.info))
+
+                # some 400 we don't recognize.  hmm.
+                raise MetawebError(u'%r' %  r)
+            raise MetawebError, 'request failed: %s: %r %r' % (url, str(e), e.fp.read())
+
+        for header in resp.info().headers:
+            log.debug('HTTP HEADER %s', header)
+            name, value = re.split("[:\n\r]", header, 1)
+            if name.lower() == 'x-metaweb-tid':
+                self.tid = value.strip()
+            #if name.lower().startswith('set-cookie'):
+            #  if value.startswith(' metaweb-user'):
+            #    pass
+            #  else:
+            #    self.saved_headers[name] = value.strip()
+            #  log.debug('HTTP VALUE %s', value)              
+
+        return resp
+
+    def _readjson(self, fp):
+        # TODO really this should be accomplished by hooking
+        # simplejson to create attrdicts instead of dicts.
+        def struct2attrdict(st):
+            """
+            copy a json structure, turning all dicts into attrdicts.
+            
+            copying descends instances of dict and list, including subclasses.
+            """
+            if isinstance(st, dict):
+                return attrdict([(k,struct2attrdict(v)) for k,v in st.items()])
+            if isinstance(st, list):
+                return [struct2attrdict(li) for li in st]
+            return st
+
+        json = fp.read()
+        if json == '':
+            log.error('the empty string is not valid json')
+            raise MetawebError('the empty string is not valid json')
+
+        try:
+            r = simplejson.loads(json)
+        except ValueError, e:
+            log.error('error parsing json string %r' % json)
+            raise MetawebError, 'error parsing JSON string: %s' % e
+
+        return struct2attrdict(r)
+
+    def _mqlresult(self, r):
+        if r.code != '/api/status/ok':
+            for msg in r.messages:
+                log.error('mqlread failed: %s %s' % (msg.code, msg.message))
+            raise MetawebError, 'query failed: %s' % r.messages[0].code
+
+        return r.result
+
+    def login(self):
+        """sign in to the service"""
+
+        assert self.username is not None
+        assert self.password is not None
+        
+        log.debug('LOGIN USERNAME: %s', self.username)
+        
+        try:
+            resp = self._httpreq('/api/account/login', 'POST',
+                                 form=dict(username=self.username,
+                                           password=self.password))
+        except urllib2.HTTPError, e:
+            raise MetawebError("login error: %s", e)
+
+        r = self._readjson(resp)
+
+        if r.code != '/api/status/ok':
+            raise MetawebError(u'%s %r' % (r.get('code',''), r.messages))
+
+        log.debug('LOGIN RESP: %r', r)
+        log.debug('LOGIN INFO: %s', resp.info())
+        log.debug('LOGIN COOKIES: %s', self._cookies)
+
+
+    def mqlreaditer(self, sq):
+        """read a structure query"""
+
+        cursor = True
+
+        while 1:
+            subq = dict(query=[sq], cursor=cursor, escape=False)
+            qstr = simplejson.dumps(dict(c0=subq))
+
+            service = '/api/service/mqlread'
+            
+            resp = self._httpreq(service, form=dict(queries=qstr))
+            r = self._readjson(resp)
+            r = r['c0']
+
+            for item in self._mqlresult(r):
+                yield item
+
+            if r['cursor']:
+                cursor = r['cursor']
+            else:
+                break
+
+    def mqlread(self, sq):
+        """read a structure query"""
+        subq = dict(query=sq, cursor=True, escape=False)
+        qstr = simplejson.dumps(dict(c0=subq))
+
+        log.debug('MQLREAD: %s', qstr)
+
+        service = '/api/service/mqlread'
+
+        resp = self._httpreq(service, form=dict(queries=qstr))
+        r = self._readjson(resp)
+        r = r['c0']
+        return self._mqlresult(r)
+
+    def trans(self, guid):
+        """translate blob from guid """
+        url = '/api/trans/raw/' + urlquote(guid)
+        resp = self._httpreq(url)
+        content = resp.read()
+        return content
+
+    def mqlwrite(self, sq):
+        """do a mql write"""
+        query = dict(query=sq, escape=False)
+        qstr = simplejson.dumps(query)
+
+        log.debug('MQLWRITE: %s', qstr)
+
+        service = '/api/service/mqlwrite'
+        resp = self._httpreq(service, 'POST',
+                             form=dict(query=qstr))
+        r = self._readjson(resp)
+        return self._mqlresult(r)
+
+    def upload(self, body, content_type):
+        """upload to the metaweb"""
+
+        log.debug('UPLOAD %s', content)
+
+        service = '/api/service/upload'
+        resp = self._httpreq(service, 'POST',
+                             headers={'content-type': content_type},
+                             body=qstr)
+        r = self._readjson(resp)
+        return self._mqlresult(r)
+
+
+if __name__ == '__main__':
+    mss = HTTPMetawebSession('sandbox.freebase.com')
+
+    print mss.mqlread([dict(name=None, type='/type/type')])
diff --git a/trunk/freebase/rison.py b/trunk/freebase/rison.py
new file mode 100755
index 0000000..468d505
--- /dev/null
+++ b/trunk/freebase/rison.py
@@ -0,0 +1,303 @@
+
+#
+# based on javascript openlaszlo-json
+#
+# Author: Oliver Steele
+# Copyright: Copyright 2006 Oliver Steele.  All rights reserved.
+# Homepage: http:#osteele.com/sources/openlaszlo/json
+# License: MIT License.
+# Version: 1.0
+#
+
+# hacked by nix for use in uris
+# ported to python by nix
+#
+
+
+
+#
+#  TODO
+#
+#  switch to unicode
+#  fall through to simplejson if first char is not in '!('
+#
+
+
+import os, sys, re
+#import simplejson
+simplejson = None
+
+class ParserException(Exception):
+    pass
+
+class Parser(object):
+    WHITESPACE = ''
+    #WHITESPACE = " \t\n\r\f"
+
+    # we divide the uri-safe glyphs into three sets
+    #   <rison> - used by rison                         ' ! : ( ) ,
+    #   <identifier> - common in strings                - _ . / ~
+    #   <reserved> - not common in strings, reserved    * @ $
+    #
+    # <rison> and <reserved> classes are illegal in ids.
+    # ids also can't look like numbers.
+
+    not_idchar  = "'!:(),*@$";
+    not_idstart = "-0123456789";
+
+    idrx = ('[^' + not_idstart + not_idchar + 
+            '][^' + not_idchar + ']*')
+
+    # regexp to check for valid rison ids
+    id_ok_re = re.compile('^' + idrx + '$', re.M)
+
+    # regexp to find the end of an id when parsing
+    next_id_re = re.compile(idrx, re.M)
+
+    def parse_json(self, str):
+        if len(str) > 0 and str[0] not in '!(':
+            return simplejson.loads(str)
+        return self.parse(str)
+
+    def parse(self, str):
+        self.string = str
+        self.index = 0
+
+        value = self.readValue()
+        if self.next():
+            raise ParserException("extra characters at the end of the string")
+        return value
+    
+    def readValue(self):
+        c = self.next()
+
+        if c == '!':
+            return self.parse_bang()
+        if c == '(':
+            return self.parse_open_paren()
+        if c == "'":
+            return self.parse_single_quote()
+        if c in '-0123456789':
+            return self.parse_number()
+
+        # fell through table, parse as an id
+        s = self.string
+        i = self.index-1
+
+        m = self.next_id_re.match(s, i)
+        if m:
+            id = m.group(0)
+            self.index = i + len(id)
+            return id  # a string
+
+        if c:
+            raise ParserException("invalid character: '" + c + "'")
+        raise ParserException("empty expression")
+
+    def parse_array(self):
+        ar = []
+        while 1:
+            c = self.next()
+            if c == ')':
+                return ar
+
+            if c is None:
+                raise ParserException("unmatched '!('")
+
+            if len(ar):
+                if c != ',':
+                    raise ParserException("missing ','")
+            elif c == ',':
+                raise ParserException("extra ','")
+            else:
+                self.index -= 1
+            n = self.readValue()
+            ar.append(n)
+
+        return ar
+
+    def parse_bang (self):
+        s = self.string
+        c = s[self.index]
+        self.index += 1
+        if c is None:
+            raise ParserException('"!" at end of input')
+        if c not in self.bangs:
+            raise ParserException('unknown literal: "!' + c + '"')
+        x = self.bangs[c]
+        if callable(x):
+            return x(self)
+
+        return x
+
+
+    def parse_open_paren (self):
+        count = 0
+        o = {}
+
+        while 1:
+            c = self.next()
+            if c == ')':
+                return o
+            if count:
+                if c != ',':
+                    raise ParserException("missing ','")
+            elif c == ',':
+                raise ParserException("extra ','")
+            else:
+                self.index -= 1
+            k = self.readValue()
+
+            if self.next() != ':':
+                raise ParserException("missing ':'")
+            v = self.readValue()
+
+            o[k] = v
+            count += 1
+        
+
+    def parse_single_quote(self):
+        s = self.string
+        i = self.index
+        start = i
+        segments = []
+
+        while 1:
+            if i >= len(s):
+                raise ParserException('unmatched "\'"')
+
+            c = s[i]
+            i += 1
+            if c == "'":
+                break
+
+            if c == '!':
+                if start < i-1:
+                    segments.append(s[start:i-1])
+                c = s[i]
+                i += 1
+                if c in "!'":
+                    segments.append(c)
+                else:
+                    raise ParserException('invalid string escape: "!'+c+'"')
+                
+                start = i
+            
+        
+        if start < i-1:
+            segments.append(s[start:i-1])
+        self.index = i
+        return ''.join(segments)
+
+
+    # Also any number start (digit or '-')
+    def parse_number(self):
+        s = self.string
+        i = self.index
+        start = i-1
+        state = 'int'
+        permittedSigns = '-'
+        transitions = {
+            'int+.': 'frac',
+            'int+e': 'exp',
+            'frac+e': 'exp'
+            }
+        
+        while 1:
+            if i >= len(s):
+                i += 1
+                break
+
+            c = s[i]
+            i += 1
+
+            if '0' <= c and c <= '9':
+                continue
+
+            if permittedSigns.find(c) >= 0:
+                permittedSigns = ''
+                continue
+
+            state = transitions.get(state + '+' + c.lower(), None)
+            if state is None:
+                break
+            if state == 'exp':
+                permittedSigns = '-'
+
+        self.index = i - 1
+        s = s[start:self.index]
+        if s == '-':
+            raise ParserException("invalid number")
+        if re.search('[.e]', s):
+            return float(s)
+        return int(s)
+    
+    # return the next non-whitespace character, or undefined
+    def next(self):
+        l = len(self.string)
+        s = self.string
+        i = self.index
+
+        while 1:
+            if i == len(s):
+                return None
+            c = s[i]
+            i += 1
+            if c not in self.WHITESPACE:
+                break
+
+        self.index = i
+        return c
+
+
+    bangs = {
+        't': True,
+        'f': False,
+        'n': None,
+        '(': parse_array
+        }
+
+
+def loads(s):
+    return Parser().parse(s)
+
+if __name__ == '__main__':
+    p = Parser()
+
+    rison_examples = [
+        "(a:0,b:1)",
+        "(a:0,b:foo,c:'23skidoo')",
+        "!t",
+        "!f",
+        "!n",
+        "''",
+        "0",
+        "1.5",
+        "-3",
+        "1e30",
+        "1e-30",
+        "G.",
+        "a",
+        "'0a'",
+        "abc def",
+        "()",
+        "(a:0)",
+        "(id:!n,type:/common/document)",
+        "!()",
+        "!(!t,!f,!n,'')",
+        "'-h'",
+        "a-z",
+        "'wow!!'",
+        "domain.com",
+        "'user@domain.com'",
+        "'US $10'",
+        "'can!'t'",
+    ];
+
+    for s in rison_examples:
+        print
+        print '*'*70
+        print
+        print s
+        
+        print '%r' % (p.parse(s),)