From 5abc781c2b09db29e76fa2ebaea9461e4e477845 Mon Sep 17 00:00:00 2001 From: mjtnix Date: Wed, 19 Sep 2007 01:51:13 +0000 Subject: remove extra trunk/ in directory path git-svn-id: http://freebase-python.googlecode.com/svn/trunk@3 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2 --- trunk/freebase/__init__.py | 2 - trunk/freebase/api/__init__.py | 2 - trunk/freebase/api/session.py | 391 ----------------------------------------- trunk/freebase/rison.py | 303 ------------------------------- 4 files changed, 698 deletions(-) delete mode 100755 trunk/freebase/__init__.py delete mode 100755 trunk/freebase/api/__init__.py delete mode 100644 trunk/freebase/api/session.py delete mode 100755 trunk/freebase/rison.py (limited to 'trunk/freebase') diff --git a/trunk/freebase/__init__.py b/trunk/freebase/__init__.py deleted file mode 100755 index d289e09..0000000 --- a/trunk/freebase/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ - -pass diff --git a/trunk/freebase/api/__init__.py b/trunk/freebase/api/__init__.py deleted file mode 100755 index 959259f..0000000 --- a/trunk/freebase/api/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ - -from session import HTTPMetawebSession, MetawebError diff --git a/trunk/freebase/api/session.py b/trunk/freebase/api/session.py deleted file mode 100644 index b3ec825..0000000 --- a/trunk/freebase/api/session.py +++ /dev/null @@ -1,391 +0,0 @@ -# ======================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB -# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# ======================================================================== - -# -# declarations for external metaweb api. -# -# -# from metaweb.api import HTTPMetawebSession -# -# mss = HTTPMetawebSession('sandbox.freebase.com') -# print mss.mqlread([dict(name=None, type='/type/type')]) -# -# -# - -import os, sys, re -import urllib2 -import cookielib -import simplejson -from urllib import quote as urlquote -import pprint -import socket - -import logging -log = logging.getLogger() -log.setLevel(logging.DEBUG) -console = logging.StreamHandler() -console.setLevel(logging.INFO) -# add the handler to the root logger -log.addHandler(console) - - -__all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession'] - -__version__ = '0.1' - -import simplejson.encoder -simplejson.JSONEncoder.item_separator = ',' -simplejson.JSONEncoder.key_separator = ':' -#delattr(simplejson.encoder.ESCAPE_DCT, '/') - -def urlencode_weak(s): - return urlquote(s, safe='{}[],"/:$') - - -# from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/361668 -class attrdict(dict): - """A dict whose items can also be accessed as member variables. - - >>> d = attrdict(a=1, b=2) - >>> d['c'] = 3 - >>> print d.a, d.b, d.c - 1 2 3 - >>> d.b = 10 - >>> print d['b'] - 10 - - # but be careful, it's easy to hide methods - >>> print d.get('c') - 3 - >>> d['get'] = 4 - >>> print d.get('a') - Traceback (most recent call last): - TypeError: 'int' object is not callable - """ - def __init__(self, *args, **kwargs): - dict.__init__(self, *args, **kwargs) - self.__dict__ = self - - - -# TODO expose the common parts of the result envelope -class MetawebError(Exception): - """ - an error report from the metaweb service. - """ - pass - - - -# TODO right now this is a completely unnecessary superclass. -# is there enough common behavior between session types -# to justify it? -class MetawebSession(object): - """ - MetawebSession is the base class for MetawebSession, subclassed for - different connection types. Only http is available externally. - - This is more of an interface than - """ - - # interface definition here... - - -class HTTPMetawebSession(MetawebSession): - """ - a MetawebSession is a request/response queue. - - this version uses the HTTP api, and is synchronous. - """ - # share cookies across sessions, so that different sessions can - # see each other's writes immediately. - _cookies = cookielib.CookieJar() - _cookiespy = urllib2.HTTPCookieProcessor(_cookies) - - def __init__(self, service_url, username=None, password=None, prev_session=None): - """ - create a new MetawebSession for interacting with the Metaweb. - - a new session will inherit state from prev_session if present, - """ - super(HTTPMetawebSession, self).__init__() - - assert not service_url.endswith('/') - if not '/' in service_url: # plain host:port - service_url = 'http://' + service_url - - self.service_url = service_url - - self.username = username - self.password = password - - self.tid = None - - if prev_session: - self.service_url = prev.service_url - - def _httpreq(self, service_path, method='GET', body=None, form=None, - headers=None, **kws): - """ - make an http request to the service. - - form arguments are encoded in the url, even for POST, if a non-form - content-type is given for the body. - """ - - if method == 'POST': - assert body is not None or form is not None - elif method == 'GET': - assert body is None - else: - assert 0, 'unknown method %s' % method - - log.debug('HTTPREQ: %s %s %s %s', service_path, method, kws, self._cookies) - - url = self.service_url + service_path - - if headers is None: - headers = [] - - # XXX This is a lousy way to parse Content-Type, where is - # the library? - ct = None - for k,v in headers: - if k.lower() == 'content-type': - ct = v.split(';')[0] - break - - if body is not None: - # if body is provided, content-type had better be too - assert ct is not None - - if form is not None: - qstr = '&'.join(['%s=%s' % (urlencode_weak(k), urlencode_weak(v)) - for k,v in form.items()]) - if method == 'POST': - # put the args on the url if we're putting something else - # in the body. this is used to add args to raw uploads. - if body is not None: - url += '?' + qstr - else: - if ct is None: - # XXX encoding and stuff - ct = 'application/x-www-form-urlencoded' - headers.append(('content-type', ct)) - - if ct == 'multipart/form-encoded': - # XXX fixme - raise NotImplementedError - elif ct == 'application/x-www-form-urlencoded': - body = qstr - else: - # for all methods other than POST, use the url - url += '?' + qstr - - req = urllib2.Request(url, body, dict(headers), **kws) - - self.opener = urllib2.build_opener(self._cookiespy) - - # assure the service that this isn't a CSRF form submission - self.opener.addheaders = [('User-agent', - 'python freebase.api-%s' % __version__), - ('X-Metaweb-Request', 'Python')] - - if self.tid is not None: - self.opener.addheaders.append(('X-Metaweb-TID', self.tid)) - - try: - if headers: - self.opener.addheaders += headers - resp = self.opener.open(req) - - except socket.error, e: - log.error('SOCKET FAILURE: %s', e.fp.read()) - raise MetawebError, 'failed contacting %s: %s' % (url, str(e)) - - except urllib2.HTTPError, e: - if e.code == 400 and e.info().type == 'application/json': - r = self._readjson(e.fp) - for msg in r.messages: - if msg.get('level', 'error') == 'error': - raise MetawebError(u'%s %s %r' % (msg.get('code',''), msg.message, msg.info)) - - # some 400 we don't recognize. hmm. - raise MetawebError(u'%r' % r) - raise MetawebError, 'request failed: %s: %r %r' % (url, str(e), e.fp.read()) - - for header in resp.info().headers: - log.debug('HTTP HEADER %s', header) - name, value = re.split("[:\n\r]", header, 1) - if name.lower() == 'x-metaweb-tid': - self.tid = value.strip() - #if name.lower().startswith('set-cookie'): - # if value.startswith(' metaweb-user'): - # pass - # else: - # self.saved_headers[name] = value.strip() - # log.debug('HTTP VALUE %s', value) - - return resp - - def _readjson(self, fp): - # TODO really this should be accomplished by hooking - # simplejson to create attrdicts instead of dicts. - def struct2attrdict(st): - """ - copy a json structure, turning all dicts into attrdicts. - - copying descends instances of dict and list, including subclasses. - """ - if isinstance(st, dict): - return attrdict([(k,struct2attrdict(v)) for k,v in st.items()]) - if isinstance(st, list): - return [struct2attrdict(li) for li in st] - return st - - json = fp.read() - if json == '': - log.error('the empty string is not valid json') - raise MetawebError('the empty string is not valid json') - - try: - r = simplejson.loads(json) - except ValueError, e: - log.error('error parsing json string %r' % json) - raise MetawebError, 'error parsing JSON string: %s' % e - - return struct2attrdict(r) - - def _mqlresult(self, r): - if r.code != '/api/status/ok': - for msg in r.messages: - log.error('mqlread failed: %s %s' % (msg.code, msg.message)) - raise MetawebError, 'query failed: %s' % r.messages[0].code - - return r.result - - def login(self): - """sign in to the service""" - - assert self.username is not None - assert self.password is not None - - log.debug('LOGIN USERNAME: %s', self.username) - - try: - resp = self._httpreq('/api/account/login', 'POST', - form=dict(username=self.username, - password=self.password)) - except urllib2.HTTPError, e: - raise MetawebError("login error: %s", e) - - r = self._readjson(resp) - - if r.code != '/api/status/ok': - raise MetawebError(u'%s %r' % (r.get('code',''), r.messages)) - - log.debug('LOGIN RESP: %r', r) - log.debug('LOGIN INFO: %s', resp.info()) - log.debug('LOGIN COOKIES: %s', self._cookies) - - - def mqlreaditer(self, sq): - """read a structure query""" - - cursor = True - - while 1: - subq = dict(query=[sq], cursor=cursor, escape=False) - qstr = simplejson.dumps(dict(c0=subq)) - - service = '/api/service/mqlread' - - resp = self._httpreq(service, form=dict(queries=qstr)) - r = self._readjson(resp) - r = r['c0'] - - for item in self._mqlresult(r): - yield item - - if r['cursor']: - cursor = r['cursor'] - else: - break - - def mqlread(self, sq): - """read a structure query""" - subq = dict(query=sq, cursor=True, escape=False) - qstr = simplejson.dumps(dict(c0=subq)) - - log.debug('MQLREAD: %s', qstr) - - service = '/api/service/mqlread' - - resp = self._httpreq(service, form=dict(queries=qstr)) - r = self._readjson(resp) - r = r['c0'] - return self._mqlresult(r) - - def trans(self, guid): - """translate blob from guid """ - url = '/api/trans/raw/' + urlquote(guid) - resp = self._httpreq(url) - content = resp.read() - return content - - def mqlwrite(self, sq): - """do a mql write""" - query = dict(query=sq, escape=False) - qstr = simplejson.dumps(query) - - log.debug('MQLWRITE: %s', qstr) - - service = '/api/service/mqlwrite' - resp = self._httpreq(service, 'POST', - form=dict(query=qstr)) - r = self._readjson(resp) - return self._mqlresult(r) - - def upload(self, body, content_type): - """upload to the metaweb""" - - log.debug('UPLOAD %s', content) - - service = '/api/service/upload' - resp = self._httpreq(service, 'POST', - headers={'content-type': content_type}, - body=qstr) - r = self._readjson(resp) - return self._mqlresult(r) - - -if __name__ == '__main__': - mss = HTTPMetawebSession('sandbox.freebase.com') - - print mss.mqlread([dict(name=None, type='/type/type')]) diff --git a/trunk/freebase/rison.py b/trunk/freebase/rison.py deleted file mode 100755 index 468d505..0000000 --- a/trunk/freebase/rison.py +++ /dev/null @@ -1,303 +0,0 @@ - -# -# based on javascript openlaszlo-json -# -# Author: Oliver Steele -# Copyright: Copyright 2006 Oliver Steele. All rights reserved. -# Homepage: http:#osteele.com/sources/openlaszlo/json -# License: MIT License. -# Version: 1.0 -# - -# hacked by nix for use in uris -# ported to python by nix -# - - - -# -# TODO -# -# switch to unicode -# fall through to simplejson if first char is not in '!(' -# - - -import os, sys, re -#import simplejson -simplejson = None - -class ParserException(Exception): - pass - -class Parser(object): - WHITESPACE = '' - #WHITESPACE = " \t\n\r\f" - - # we divide the uri-safe glyphs into three sets - # - used by rison ' ! : ( ) , - # - common in strings - _ . / ~ - # - not common in strings, reserved * @ $ - # - # and classes are illegal in ids. - # ids also can't look like numbers. - - not_idchar = "'!:(),*@$"; - not_idstart = "-0123456789"; - - idrx = ('[^' + not_idstart + not_idchar + - '][^' + not_idchar + ']*') - - # regexp to check for valid rison ids - id_ok_re = re.compile('^' + idrx + '$', re.M) - - # regexp to find the end of an id when parsing - next_id_re = re.compile(idrx, re.M) - - def parse_json(self, str): - if len(str) > 0 and str[0] not in '!(': - return simplejson.loads(str) - return self.parse(str) - - def parse(self, str): - self.string = str - self.index = 0 - - value = self.readValue() - if self.next(): - raise ParserException("extra characters at the end of the string") - return value - - def readValue(self): - c = self.next() - - if c == '!': - return self.parse_bang() - if c == '(': - return self.parse_open_paren() - if c == "'": - return self.parse_single_quote() - if c in '-0123456789': - return self.parse_number() - - # fell through table, parse as an id - s = self.string - i = self.index-1 - - m = self.next_id_re.match(s, i) - if m: - id = m.group(0) - self.index = i + len(id) - return id # a string - - if c: - raise ParserException("invalid character: '" + c + "'") - raise ParserException("empty expression") - - def parse_array(self): - ar = [] - while 1: - c = self.next() - if c == ')': - return ar - - if c is None: - raise ParserException("unmatched '!('") - - if len(ar): - if c != ',': - raise ParserException("missing ','") - elif c == ',': - raise ParserException("extra ','") - else: - self.index -= 1 - n = self.readValue() - ar.append(n) - - return ar - - def parse_bang (self): - s = self.string - c = s[self.index] - self.index += 1 - if c is None: - raise ParserException('"!" at end of input') - if c not in self.bangs: - raise ParserException('unknown literal: "!' + c + '"') - x = self.bangs[c] - if callable(x): - return x(self) - - return x - - - def parse_open_paren (self): - count = 0 - o = {} - - while 1: - c = self.next() - if c == ')': - return o - if count: - if c != ',': - raise ParserException("missing ','") - elif c == ',': - raise ParserException("extra ','") - else: - self.index -= 1 - k = self.readValue() - - if self.next() != ':': - raise ParserException("missing ':'") - v = self.readValue() - - o[k] = v - count += 1 - - - def parse_single_quote(self): - s = self.string - i = self.index - start = i - segments = [] - - while 1: - if i >= len(s): - raise ParserException('unmatched "\'"') - - c = s[i] - i += 1 - if c == "'": - break - - if c == '!': - if start < i-1: - segments.append(s[start:i-1]) - c = s[i] - i += 1 - if c in "!'": - segments.append(c) - else: - raise ParserException('invalid string escape: "!'+c+'"') - - start = i - - - if start < i-1: - segments.append(s[start:i-1]) - self.index = i - return ''.join(segments) - - - # Also any number start (digit or '-') - def parse_number(self): - s = self.string - i = self.index - start = i-1 - state = 'int' - permittedSigns = '-' - transitions = { - 'int+.': 'frac', - 'int+e': 'exp', - 'frac+e': 'exp' - } - - while 1: - if i >= len(s): - i += 1 - break - - c = s[i] - i += 1 - - if '0' <= c and c <= '9': - continue - - if permittedSigns.find(c) >= 0: - permittedSigns = '' - continue - - state = transitions.get(state + '+' + c.lower(), None) - if state is None: - break - if state == 'exp': - permittedSigns = '-' - - self.index = i - 1 - s = s[start:self.index] - if s == '-': - raise ParserException("invalid number") - if re.search('[.e]', s): - return float(s) - return int(s) - - # return the next non-whitespace character, or undefined - def next(self): - l = len(self.string) - s = self.string - i = self.index - - while 1: - if i == len(s): - return None - c = s[i] - i += 1 - if c not in self.WHITESPACE: - break - - self.index = i - return c - - - bangs = { - 't': True, - 'f': False, - 'n': None, - '(': parse_array - } - - -def loads(s): - return Parser().parse(s) - -if __name__ == '__main__': - p = Parser() - - rison_examples = [ - "(a:0,b:1)", - "(a:0,b:foo,c:'23skidoo')", - "!t", - "!f", - "!n", - "''", - "0", - "1.5", - "-3", - "1e30", - "1e-30", - "G.", - "a", - "'0a'", - "abc def", - "()", - "(a:0)", - "(id:!n,type:/common/document)", - "!()", - "!(!t,!f,!n,'')", - "'-h'", - "a-z", - "'wow!!'", - "domain.com", - "'user@domain.com'", - "'US $10'", - "'can!'t'", - ]; - - for s in rison_examples: - print - print '*'*70 - print - print s - - print '%r' % (p.parse(s),) -- cgit v1.3.1