diff options
Diffstat (limited to 'freebase/fcl')
| -rw-r--r-- | freebase/fcl/README.txt | 8 | ||||
| -rwxr-xr-x | freebase/fcl/__init__.py | 0 | ||||
| -rwxr-xr-x | freebase/fcl/commands.py | 688 | ||||
| -rwxr-xr-x | freebase/fcl/fbutil.py | 113 | ||||
| -rwxr-xr-x | freebase/fcl/fcl.py | 276 | ||||
| -rwxr-xr-x | freebase/fcl/inspect.py | 218 | ||||
| -rwxr-xr-x | freebase/fcl/mktype.py | 179 |
7 files changed, 1482 insertions, 0 deletions
diff --git a/freebase/fcl/README.txt b/freebase/fcl/README.txt new file mode 100644 index 0000000..2e03d71 --- /dev/null +++ b/freebase/fcl/README.txt @@ -0,0 +1,8 @@ + +This is a command-line tool for performing various freebase commands. + +It should be installed by default as "fcl". + +Try "fcl help" after installing the freebase-api library. + + diff --git a/freebase/fcl/__init__.py b/freebase/fcl/__init__.py new file mode 100755 index 0000000..e69de29 --- /dev/null +++ b/freebase/fcl/__init__.py diff --git a/freebase/fcl/commands.py b/freebase/fcl/commands.py new file mode 100755 index 0000000..0fdc283 --- /dev/null +++ b/freebase/fcl/commands.py @@ -0,0 +1,688 @@ +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + +import os, sys, re, time +from fbutil import * + +import simplejson + +import freebase.rison as rison +from freebase.api import attrdict + + +def cmd_help(fb, command=None): + """get help on commands + + %prog help [cmd] + """ + + if command is not None: + cmd = fb.commands[command] + print ' %s: %s' % (cmd.name, cmd.shortdoc) + print cmd.doc + return + + print """ +the interface to this tool is loosely modeled on the +"svn" command-line tool. many commands work slightly +differently from their unix and svn equivalents - +read the doc for the command first. + +use "%s help <subcommand>" for help on a particular +subcommand. +""" % fb.progpath + + fb.oparser.print_help() + + print 'available subcommands:' + cmds = sorted(fb.commands.keys()) + + for fk in cmds: + cmd = fb.commands[fk] + print ' %s: %s' % (cmd.name, cmd.shortdoc) + + +def cmd_wikihelp(fb): + """get help on commands in mediawiki markup format + + %prog wikihelp + """ + + print """usage: %s subcommand [ args ... ] + + the interface to this tool is loosely modeled on the + "svn" command-line tool. many commands work slightly + differently from their unix and svn equivalents - + read the doc for the command first. + """ % fb.progpath + + print 'subcommands:' + for fk in sorted(fb.commands.keys()): + cmd = fb.commands[fk] + + # mediawiki markup cares about the difference between + # a blank line and a blank line with a bunch of + # spaces at the start of it. ewww. + doc = re.sub(r'\r?\n {0,8}', '\n ', cmd.doc) + + print '==== %s %s: %s ====' % (fb.progpath, cmd.name, cmd.shortdoc) + print doc + print + + +# this command is disabled because it relies on some svn glue that +# was not sufficiently well thought out. +def old_cmd_pwid(fb): + """show the "current working namespace id" + + %prog pwid + + by default, relative ids can't be resolved. however, if you + run the fb tool from a directory that has the svn property + 'freebase:id' set, relative ids will be resolved relative + to that id. the idea is that you can create an svn tree + that parallels the freebase namespace tree. + + for example: + $ %prog pwid + + $ svn propset freebase:id "/freebase" . + + $ %prog pwid + /freebase + + """ + if fb.cwid is not None: + print fb.cwid + else: + print '' + + +def cmd_ls(fb, path=None): + """list the keys in a namespace + + %prog ls [id] + """ + path = fb.absid(path) + q = {'id': path, + '/type/namespace/keys': [{'value': None, + 'namespace': { + 'id': None, + 'type': [] + }, + 'optional':True + }] + } + + r = fb.mss.mqlread(q) + if r is None: + raise CmdException('query for id %r failed' % path) + + #sys.stdout.write(' '.join([mk.value for mk in r['/type/namespace/keys']])) + + for mk in r['/type/namespace/keys']: + print mk.value + + if 0: + suffix = '' + if ('/type/namespace' in mk.namespace.type + or '/type/domain' in mk.namespace.type): + suffix = '/' + print mk.value, mk.namespace.id+suffix + + +def cmd_mkdir(fb, path): + """create a new freebase namespace + %prog mkdir id + + create a new instance of /type/namespace at the given + point in id space. if id already exists, it should be + a namespace. + """ + path = fb.absid(path) + dir,file = dirsplit(path) + wq = { 'create': 'unless_exists', + 'key':{ + 'connect': 'insert', + 'namespace': dir, + 'value': file + }, + 'name': path, + 'type': '/type/namespace' + } + + r = fb.mss.mqlwrite(wq) + +def cmd_ln(fb, src, dst): + """create a namespace key + %prog ln srcid dstid + + create a new namespace link at dstid to the object + currently at srcid. + """ + src = fb.absid(src) + dst = fb.absid(dst) + dir,file = dirsplit(dst) + wq = { 'id': src, + 'key':{ + 'connect': 'insert', + 'namespace': dir, + 'value': file + } + } + + r = fb.mss.mqlwrite(wq) + + +def cmd_rm(fb, path): + """unlink a namespace key + %prog rm id + + remove the /type/key that connects the given id to its + parent. id must be a path for this to make any sense. + + note that is like unix 'unlink' rather than 'rm'. + it won't complain if the 'subdirectory' contains data, + since that data will still be accessible to other queries. + it's not like 'rm -rf' either, because it doesn't + disturb anything other than the one directory entry. + """ + path = fb.absid(path) + dir,file = dirsplit(path) + + wq = { 'id': path, + 'key':{ + 'connect': 'delete', + 'namespace': dir, + 'value': file + } + } + + r = fb.mss.mqlwrite(wq) + +def cmd_mv(fb, src, dst): + """rename srcid to dstid. + %prog mv srcid dstid + + equivalent to: + $ fb ln <srcid> <dstid> + $ fb rm <srcid> + """ + cmd_ln(fb, src, dst) + cmd_rm(fb, src) + +def cmd_cat(fb, id, include_headers=False): + """download a document from freebase to stdout + %prog cat id + + equivalent to "%prog get id -". + """ + return cmd_get(fb, id, localfile='-', include_headers=include_headers) + +def cmd_get(fb, id, localfile=None, include_headers=False): + """download a file from freebase + %prog get id [localfile] + + download the document or image with the given id from freebase + into localfile. localfile '-' means stdout. localfile + defaults to a file in the current directory with the same name + as the last key in the path, possibly followed by a metadata + extension like .html or .txt. + """ + id = fb.absid(id) + dir,file = dirsplit_unsafe(id) + + def read_content(id, content_only=False): + c = attrdict(id=id) + cq = { 'id': id, + 'type': [], + '/common/document/content': None, + '/common/document/source_uri': None, + '/type/content/media_type': { 'name':None, + 'optional': True }, + #'/type/content/text_encoding': { 'name':None }, + '/type/content/blob_id':None, + } + cd = fb.mss.mqlread(cq) + if '/type/content' in cd.type: + c.media_type = cd['/type/content/media_type'].name + #c.text_encoding = cd['/type/content/text_encoding'].name + c.sha256 = cd['/type/content/blob_id'] + return c + + if content_only: + raise CmdException('%s is not a content id' % id) + + cid = cd['/common/document/content'] + if cid is not None: + return read_content(cid, content_only=True) + + # in this case we don't have a content object + if cd['/common/document/source_uri'] is not None: + return None + + raise CmdException('%s is not a content or document id' % id) + + + content = read_content(id) + log.debug('COBJ %r' % content) + if content is not None: + fileext = media_type_to_extension.get(content.media_type, None) + else: + fileext = None + + if localfile == '-': + ofp = sys.stdout + else: + if localfile is None: + implicit_outfile = True + localfile = file + elif re.match(r'[/\\]$', localfile): + implicit_outfile = True + localfile = localfile + file + else: + implicit_outfile = False + localfile = os.path.abspath(localfile) + + # add file extension based on content-type: + # should be an option to disable this + if implicit_outfile and fileext is not None: + localfile += '.' + fileext + + # if we didn't explicitly name the output file, + # don't destroy an existing file + localfile_base = localfile + count = 0 + while implicit_outfile and os.path.exists(localfile): + count += 1 + localfile = '%s.%d' % (localfile_base, count) + ofp = open(localfile, 'wb') + + body = fb.mss.trans(id) + + if include_headers: + # XXX show content-type, what else? + pass + + ofp.write(body) + + if localfile != '-': + print ('%s saved (%d bytes)' % (localfile, len(body))) + ofp.close() + + +def cmd_put(fb, localfile, id=None, content_type=None): + """upload a document to freebase -- EXPERIMENTAL + %prog put localfile [id] [content-type] + + upload the document or image in localfile to given freebase + id. if localfile is '-' the data will be read from stdin. + + if id is missing or empty, a new document will be created. + later the id might default to something computed from localfile + and any svn attributes it has. + + output: a single line, the id of the document. + """ + if content_type is None: + ext = re.sub('^.*\.([^/.]+)$', r'\1', localfile) + media_type = extension_to_media_type.get(ext, None) + + if media_type is None: + raise CmdException('could not infer a media type from extension %r: please specify it' + % ext) + + if media_type.startswith('text/'): + # this is a bad assumption. should sniff it? + text_encoding = 'utf-8' + content_type = '%s;charset=%s' % (media_type, text_encoding) + else: + content_type = media_type + + new_id = None + if id is not None: + idinfo = fb.mss.mqlread({ 'id': id, 'type': '/common/document' }) + if idinfo is None: + new_id = id + id = None + + body = open(localfile, 'rb').read() + r = fb.mss.upload(body, content_type, document_id=id) + + if new_id is None: + print r.document + else: + cmd_ln(fb, r.document, new_id) + print new_id + + +def cmd_dump(fb, id): + """show all properties of a freebase object + %prog dump object_id + """ + id = fb.absid(id) + + import inspect + + r = inspect.inspect_object(fb.mss, id) + if r is None: + raise CmdException('no match for id %r' % id) + + for k in sorted(r.keys()): + vs = r[k] + for v in vs: + id = v.get('id', '') + + name = '%r' % (v.get('name') or v.get('value')) + if name == 'None': name = '' + + type = v.get('type', '') + if type == '/type/text': + extra = v.get('lang', '') + elif type == '/type/key': + extra = v.get('namespace', '') + else: + extra = '' + + fb.trow(k, id, name, type, extra) + + +def cmd_pget(fb, id, propid): + """get a property of a freebase object -- EXPERIMENTAL + %prog pget object_id property_id + + get the property named by property_id from the object. + + XXX output quoting is not well specified. + + property_id must be a fully qualified id for now. + + prints one line for each match. + + if propid ends in '*' this does a wildcard for a particular type. + """ + id = fb.absid(id) + proptype, propkey = dirsplit(propid) + + if propkey != '*': + # look up the prop + q = { 'id': id, + propid: [{}], + } + r = fb.mss.mqlread(q) + for v in r[propid]: + if 'value' in v: + print v.value + else: + print v.id + + else: + # look up the prop + q = { 'id': id, + '*': [{}], + } + if isinstance(proptype, basestring): + q['type'] = proptype + + r = fb.mss.mqlread(q) + + for k in sorted(r.keys()): + v = r[k]; + if 'value' in v: + print '%s %s' % (k, v.value) + else: + print '%s %s' % (k, v.id) + +def cmd_pdel(fb, id, propid, oldval): + """delete a property of a freebase object -- EXPERIMENTAL + %prog pdel object_id property_id oldvalue + + set the property named by property_id on the object. + value is an id or a json value. XXX this is ambiguous. + + property_id must be a fully qualified id for now. + + for now you need to provide a "oldval" argument, + later this tool will query and perhaps prompt if the + deletion is ambiguous. + + prints a single line, either 'deleted' or 'missing' + """ + return cmd_pset(fb, id, propid, None, oldval) + + +def cmd_touch(fb): + """bypass any cached query results the service may have. use sparingly. + """ + fb.mss.mqlflush() + + +def cmd_pset(fb, id, propkey, val, oldval=None, extra=None): + """set a property of a freebase object -- EXPERIMENTAL + %prog pset object_id property_id value + + set the property named by property_id on the object. + value is an id or a json value. XXX this is ambiguous. + + property_id must be a fully qualified id for now. + + if the property should be a unique property, this will + write with 'connect:update'. if the property may have + multiple, it is written with 'connect:insert'. + + prints a single line, either 'inserted' or 'present' + """ + id = fb.absid(id) + + propid = fb.absprop(propkey) + + # look up the prop + pq = { 'id': propid, + 'type': '/type/property', + 'name': None, + 'unique': None, + 'expected_type': { + 'id': None, + 'name': None, + 'default_property': None, + 'optional': True, + }, + } + + prop = fb.mss.mqlread(pq) + + if prop is None: + raise CmdException('can\'t resolve property key %r - use an absolute id' % propid); + + if propid.startswith('/type/object/') or propid.startswith('/type/value/'): + propkey = re.sub('/type/[^/]+/', '', propid); + else: + propkey = propid + + wq = { 'id': id, + propkey: { + } + } + + if val is None: + val = oldval + wq[propkey]['connect'] = 'delete' + elif prop.unique: + wq[propkey]['connect'] = 'update' + else: + wq[propkey]['connect'] = 'insert' + + if prop.expected_type is None: + wq[propkey]['id'] = val + elif prop.expected_type.id not in value_types: + wq[propkey]['id'] = val + else: + wq[propkey]['value'] = val + + if prop.expected_type.id == '/type/text': + if extra is not None: + lang = extra + else: + lang = '/lang/en' + wq[propkey]['lang'] = lang + + if prop.expected_type.id == '/type/key': + if extra is not None: + wq[propkey]['namespace'] = extra + else: + raise CmdException('must specify a namespace to pset /type/key') + + r = fb.mss.mqlwrite(wq) + print r[propkey]['connect'] + +def cmd_login(fb, username=None, password=None): + """login to the freebase service + %prog login [username [password]] + + cookies are maintained in a file so + they are available to the next invocation. + prompts for username and password if not given + """ + import getpass + if username is None: + sys.stdout.write('freebase.com username: ') + username = sys.stdin.readline() + if not username: + raise CmdException('usernmae required for login') + username = re.compile('\n$').sub('', username) + if password is None: + password = getpass.getpass('freebase.com password: ') + + fb.mss.username = username + fb.mss.password = password + fb.mss.login() + +def cmd_logout(fb): + """logout from the freebase service + %prog logout + + deletes the login cookies + """ + fb.cookiejar.clear(domain=fb.service_host.split(':')[0]) + + +def cmd_find(fb, qstr): + """print all ids matching a given constraint. + + if the query string starts with "{" it is treated as json. + otherwise it is treated as o-rison. + + %prog find + """ + if qstr.startswith('{'): + q = simplejson.loads(qstr) + else: + q = rison.loads('(' + qstr + ')') + + if 'id' not in q: + q['id'] = None + + results = fb.mss.mqlreaditer(q) + for r in results: + print r.id + +def cmd_q(fb, qstr): + """run a freebase query. + + if the query string starts with "{" it is treated as json. + otherwise it is treated as o-rison. + + dump the result as json. + + %prog q + """ + if qstr.startswith('{'): + q = simplejson.loads(qstr) + else: + q = rison.loads('(' + qstr + ')') + + # results could be streamed with a little more work + results = fb.mss.mqlreaditer(q) + print simplejson.dumps(list(results), indent=2) + +def cmd_open(fb, id): + """open a web browser on the given id. works on OSX only for now. + + %prog open /some/id + """ + os.system("open 'http://www.freebase.com/view%s'" % id) + + +def cmd_log(fb, id): + """log changes pertaining to a given id. + + INCOMPLETE + + %prog log /some/id + """ + + null = None + true = True + false = False + + baseq = { + 'type': '/type/link', + 'source': null, + 'master_property': null, + 'attribution': null, + 'timestamp': null, + 'operation': null, + 'valid': null, + 'sort': '-timestamp' + }; + + queries = [ + { + 'target_value': { '*': null }, + 'target': { 'id': null, 'name': null, 'optional': true }, + }, + { + 'target': { 'id': null, 'name': null }, + }] + + for i,q in list(enumerate(queries)): + q.update(baseq) + queries[i] = [q] + + + valuesfrom,linksfrom = fb.mss.mqlreadmulti(queries) + + for link in linksfrom: + # fb.trow(link.master_property.id, ...) + print simplejson.dumps(link, indent=2) + + for link in valuesfrom: + # fb.trow(link.master_property.id, ...) + print simplejson.dumps(link, indent=2) + + diff --git a/freebase/fcl/fbutil.py b/freebase/fcl/fbutil.py new file mode 100755 index 0000000..ed1e511 --- /dev/null +++ b/freebase/fcl/fbutil.py @@ -0,0 +1,113 @@ +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + + +import re + +import logging +log = logging.getLogger() + + +class FbException(Exception): + pass + +class CmdException(Exception): + pass + +media_types = { + 'html': ['text/html'], + 'txt': ['text/plain'], + + 'xml': ['text/xml', + 'application/xml'], + + 'atom': ['application/atom+xml'], + + 'js': ['text/javascript', + 'application/javascript', + 'application/x-javascript'], + + 'json': ['application/json'], + + 'jpg': ['image/jpeg', + 'image/pjpeg'], + + 'gif': ['image/gif'], + + 'png': ['image/png'], + } + +extension_to_media_type = dict([(k,vs[0]) for k,vs in media_types.items()]) +media_type_to_extension = {} +for k,vs in media_types.items(): + for v in vs: + media_type_to_extension[v] = k + + + + +DIRSPLIT = re.compile(r'^(.+)/([^/]+)$') + +def dirsplit_unsafe(id): + m = DIRSPLIT.match(id) + if m is None: + return (None, id) + dir,file = m.groups() + return (dir,file) + +def dirsplit(id): + dir,file = dirsplit_unsafe(id) + if dir == '/guid': + raise FbException('%r is not a freebase keypath' % (id,)) + return (dir,file) + +value_types = [ + '/type/text', + '/type/key', + '/type/rawstring', + '/type/float', + '/type/int', + '/type/boolean', + '/type/uri', + '/type/datetime', + '/type/id', + '/type/enumeration', + ] + +default_propkeys = { + 'value': '/type/value/value', + 'id': '/type/object/id', + 'guid': '/type/object/guid', + 'type': '/type/object/type', + 'name': '/type/object/name', + 'key': '/type/object/key', + 'timestamp': '/type/object/timestamp', + 'permission': '/type/object/permission', + 'creator': '/type/object/creator', + 'attribution': '/type/object/attribution' +}; diff --git a/freebase/fcl/fcl.py b/freebase/fcl/fcl.py new file mode 100755 index 0000000..a0299b8 --- /dev/null +++ b/freebase/fcl/fcl.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + +import os, sys, re, time, stat + +from optparse import OptionParser +import getpass +import cookielib +import logging +import simplejson + + +from fbutil import FbException, CmdException, log, default_propkeys +console = logging.StreamHandler() +log.addHandler(console) + +from freebase.api import HTTPMetawebSession, MetawebError, attrdict + +_cookiedir = None +if os.environ.has_key('HOME'): + _cookiedir = os.path.join(os.environ['HOME'], '.pyfreebase') + + +class Command(object): + def __init__(self, module, name, func): + self.module = module + self.name = name + self.func = func + + PROG = re.compile(r'\%prog') + NEWLINE = re.compile(r'(?:\r?\n)+') + + # fill in self.doc + if isinstance(func.__doc__, basestring): + doc = PROG.sub('fcl', func.__doc__ + '\n') + self.shortdoc, self.doc = NEWLINE.split(doc, 1) + else: + self.shortdoc = '(missing documentation)' + self.doc = '(missing documentation)' + + + +class FbCommandHandler(object): + + def __init__(self): + self.service_host = 'www.freebase.com' + self.cookiejar = None + self.cwid = '' + self.progpath = 'fcl' + self.commands = {} + + self.cookiefile = None + if _cookiedir is not None: + self.cookiefile = os.path.join(_cookiedir, 'cookiejar') + + + def init(self): + if self.cookiefile is not None: + self.cookiejar = cookielib.LWPCookieJar(self.cookiefile) + if os.path.exists(self.cookiefile): + try: + self.cookiejar.load(ignore_discard=True) + except cookielib.LoadError: + log.warn('error loading cookies') + + #print 'start cookies %r' % self.cookiejar + + self.mss = HTTPMetawebSession(self.service_host, + cookiejar=self.cookiejar) + + + def absid(self, path): + if path is None: + path = '' + if path.startswith('/'): + return path + + if not isinstance(self.cwid, basestring) or not self.cwid.startswith('/'): + # svn cwid support is disabled because it relies on some svn glue that + # was not sufficiently well thought out. + # raise CmdException("can't resolve relative id %r without cwid - see 'fcl help pwid'" % (path)) + raise CmdException("no support for relative id %r" % (path)) + + if path == '' or path == '.': + return self.cwid + + return self.cwid + '/' + path + + + def absprop(self, propkey): + if propkey.startswith('/'): + return propkey + + # check schemas of /type/object and /type/value, + # as well as other reserved names + if propkey in default_propkeys: + return default_propkeys[propkey] + + return self.absid(propkey) + + def thead(self, *args): + strs = ['%r' % arg + for arg in args] + print '\t'.join(strs) + + def trow(self, *args): + print '\t'.join(args) + return + strs = ['%r' % arg + for arg in args] + print '\t'.join(strs) + + def save(self): + #print 'end cookies %r' % self.cookiejar + if _cookiedir and self.cookiefile.startswith(_cookiedir): + # create private cookiedir if needed + if not os.path.exists(_cookiedir): + os.mkdir(_cookiedir, 0700) + os.chmod(_cookiedir, stat.S_IRWXU) + + if self.cookiejar is None: + return + + self.cookiejar.save(ignore_discard=True) + + # save the cwd and other state too + + + def import_commands(self, modname): + """ + import new fb commands from a file + """ + namespace = {} + + pyimport = 'from %s import *' % modname + exec pyimport in namespace + mod = sys.modules.get(modname) + + commands = [Command(mod, k[4:], getattr(mod, k)) + for k in getattr(mod, '__all__', dir(mod)) + if (k.startswith('cmd_') + and callable(getattr(mod, k)))] + + + for cmd in commands: + log.info('importing %r' % ((cmd.name, cmd.func),)) + self.commands[cmd.name] = cmd + + log.info('imported %r from %r' % (modname, mod.__file__)) + + + def dispatch(self, cmd, args): + if cmd in self.commands: + try: + self.commands[cmd].func(self, *args) + except KeyboardInterrupt, e: + sys.stderr.write('%s\n' % (str(e),)) + except FbException, e: + sys.stderr.write('%s\n' % (str(e),)) + except CmdException, e: + sys.stderr.write('%s\n' % (str(e),)) + except MetawebError, e: + sys.stderr.write('%s\n' % (str(e),)) + else: + self.oparser.error('unknown subcommand %r, try "%s help"' % (cmd, self.progpath)) + + self.save() + + + def cmdline_main(self): + op = OptionParser(usage='%prog [options] command [args...] ') + self.oparser = op + + op.disable_interspersed_args() + + op.add_option('-d', '--debug', dest='debug', + default=False, action='store_true', + help='turn on debugging output') + + op.add_option('-v', '--verbose', dest='verbose', + default=False, action='store_true', + help='verbose output') + + op.add_option('-V', '--very-verbose', dest='very_verbose', + default=False, action='store_true', + help='lots of debug output') + + op.add_option('-s', '--service', dest='service_host', + metavar='HOST', + default=self.service_host, + help='Freebase HTTP service address:port') + + op.add_option('-S', '--sandbox', dest='use_sandbox', + default=False, action='store_true', + help='shortcut for --service=sandbox.freebase.com') + + op.add_option('-c', '--cookiejar', dest='cookiefile', + metavar='FILE', + default=self.cookiefile, + help='Cookie storage file (will be created if missing)') + options,args = op.parse_args() + + if len(args) < 1: + op.error('required subcommand missing') + + + loglevel = logging.WARNING + if options.verbose: + loglevel = logging.INFO + if options.very_verbose: + loglevel = logging.DEBUG + + console.setLevel(loglevel) + log.setLevel(loglevel) + + if options.use_sandbox: + self.service_host = 'sandbox.freebase.com' + else: + self.service_host = options.service_host + + self.cookiefile = options.cookiefile + #self.progpath = sys.argv[0] + + self.init() + + self.mss.log.setLevel(loglevel) + self.mss.log.addHandler(console) + + self.import_commands('freebase.fcl.commands') + self.import_commands('freebase.fcl.mktype') + + cmd = args.pop(0) + self.dispatch(cmd, args) + +# entry point for script +def main(): + try: + # turn off crlf output on windows so we work properly + # with unix tools. + import msvcrt + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) + except ImportError: + pass + + fb = FbCommandHandler() + fb.cmdline_main() + +if __name__ == '__main__': + main() diff --git a/freebase/fcl/inspect.py b/freebase/fcl/inspect.py new file mode 100755 index 0000000..8cd04d2 --- /dev/null +++ b/freebase/fcl/inspect.py @@ -0,0 +1,218 @@ +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + + +# +# +# wrap all the nastiness needed for a general mql inspect query +# +# + +import os, sys, re + + +null = None +true = True +false = False + +inspect_query = { + 'name': null, + 'type': [], + + '/type/reflect/any_master': [{ + 'optional':true, + 'id': null, + 'name': null, + 'link': { + 'master_property': { + 'id': null, + 'schema': null + } + } + }], + + '/type/reflect/any_reverse': [{ + 'optional':true, + 'id': null, + 'name': null, + 'link': { + 'master_property': { + 'id':null, + 'schema': null, + 'expected_type': null, + 'reverse_property': { + 'id': null, + 'schema': null, + 'optional': true + } + } + } + }], + + '/type/reflect/any_value': [{ + 'optional':true, + 'value': null, + 'link': { + 'master_property': { + 'id':null, + 'schema': null, + 'expected_type': null + }, + } + }], + + 't:/type/reflect/any_value': [{ + 'optional':true, + 'type': '/type/text', + 'value': null, + 'lang': null, + 'link': { + 'master_property': { + 'id':null, + 'schema': null + }, + } + }], + + '/type/object/creator': [{ + 'optional':true, + 'id':null, + 'name':null + }], + '/type/object/timestamp': [{ + 'optional':true, + 'value': null, + }], + + '/type/object/key': [{ + 'optional':true, + 'value': null, + 'namespace': null + }], + '/type/namespace/keys': [{ + 'optional':true, + 'value': null, + 'namespace': null + }] +} + + +def transform_result(result): + proptypes = {} + props = {} + + # copy a property from a /type/reflect clause + def pushtype(propdesc, prop): + tid = propdesc['schema'] + propid = propdesc['id'] + + if isinstance(prop, dict): + prop = dict(prop) + if 'link' in prop: + prop.pop('link') + + if tid not in proptypes: + proptypes[tid] = {} + if propid not in proptypes[tid]: + proptypes[tid][propid] = [] + + if propid not in props: + props[propid] = [] + props[propid].append(prop) + + # copy a property that isn't enumerated by /type/reflect + def pushprop(propid): + ps = result[propid] + if ps is None: + return + + # hack to infer the schema from id, not always reliable! + schema = re.sub(r'/[^/]+$', '', propid) + keyprop = dict(id=propid, schema=schema) + for p in ps: + pushtype(keyprop, p) + + ps = result['/type/reflect/any_master'] or [] + for p in ps: + propdesc = p.link.master_property + pushtype(propdesc, p) + + # non-text non-key values + ps = result['/type/reflect/any_value'] or [] + for p in ps: + propdesc = p.link.master_property + + # /type/text values are queried specially + # so that we can get the lang, so ignore + # them here. + if propdesc.expected_type == '/type/text': + continue + + pushtype(propdesc, p) + + # text values + ps = result['t:/type/reflect/any_value'] or [] + for p in ps: + propdesc = p.link.master_property + pushtype(propdesc, p) + + pushprop('/type/object/creator') + pushprop('/type/object/timestamp') + pushprop('/type/object/key') + pushprop('/type/namespace/keys') + + # now the reverse properties + ps = result['/type/reflect/any_reverse'] or [] + for prop in ps: + propdesc = prop.link.master_property.reverse_property + + # synthetic property descriptor for the reverse of + # a property with no reverse descriptor. + # note the bogus id starting with '-'. + if propdesc is None: + # schema = prop.link.master_property.expected_type + # if schema is None: + # schema = 'other' + + schema = 'other' + propdesc = dict(id='-' + prop.link.master_property.id, + schema=schema) + + pushtype(propdesc, prop) + + #return proptypes + return props + + +def inspect_object(mss, id): + q = dict(inspect_query) + q['id'] = id + r = mss.mqlread(q) + if r is None: + return None + return transform_result(r) diff --git a/freebase/fcl/mktype.py b/freebase/fcl/mktype.py new file mode 100755 index 0000000..30f8df7 --- /dev/null +++ b/freebase/fcl/mktype.py @@ -0,0 +1,179 @@ +# ================================================================== +# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB +# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ==================================================================== + +import os, sys, re, time +from fbutil import * + + +def cmd_mkobj(fb, id, typeid='/common/topic', name=''): + """create a new object with a given type -- EXPERIMENTAL + %prog mkobj new_id typeid name + + create a new object with type typeid at the given + namespace location. + + if present, name gives the display name of the new object. + + """ + id = fb.absid(id) + nsid, key = dirsplit(id) + + typeid = fb.absid(typeid) + + if name == '': + name = key + + wq = { 'create': 'unless_exists', + 'id': None, + 'name': name, + 'type': typeid, + 'key':{ + 'namespace': nsid, + 'value': key + }, + } + + # TODO add included types + + r = fb.mss.mqlwrite(wq) + print r.id,r.create + +def cmd_mktype(fb, id, name=''): + """create a new type -- EXPERIMENTAL + %prog mktype new_id name + + create a new object with type Type at the given + namespace location. + + this doesn't create any type hints. + + if present, name gives the display name of the new property + """ + id = fb.absid(id) + + nsid, key = dirsplit(id) + + if name == '': + name = key + + wq = { 'create': 'unless_exists', + 'id': None, + 'name': name, + 'type': '/type/type', + 'key':{ + 'namespace': nsid, + 'value': key + }, + } + + r = fb.mss.mqlwrite(wq) + print r.id,r.create + +def mkprop(fb, typeid, key, name='', vtype=None, master_property=None): + """helper to create a new property + """ + if name == '': + name = key + + wq = { 'create': 'unless_exists', + 'id': None, + 'type': '/type/property', + 'name': name, + 'schema': typeid, + 'key': { + 'namespace': typeid, + 'value': key + } + } + + if vtype is not None: + wq['expected_type'] = vtype + if master_property is not None: + wq['master_property'] = master_property + + return fb.mss.mqlwrite(wq) + + +def cmd_mkprop(fb, id, name='', vtype=None, revkey=None, revname=''): + """create a new property -- EXPERIMENTAL + %prog mkprop new_id [name] [expected_type] [reverse_property] [reverse_name] + + create a new object with type Property at the given + location. creates both the "schema" and "key" links + for the property, but doesn't create any freebase property + hints. + + if present, name gives the display name of the new property + """ + id = fb.absid(id) + if vtype is not None: + vtype = fb.absid(vtype) + + typeid, key = dirsplit(id) + + + r = mkprop(fb, typeid, key, name, vtype) + + # write the reverse property if specified + + print r.id, r.create + + if revkey is None: + return + + assert vtype is not None + + rr = mkprop(fb, vtype, revkey, revname, typeid, id) + print rr.id, rr.create + + +def cmd_publish_type(fb, typeid): + """try to publish a freebase type for the client + %prog publish_type typeid + + set /freebase/type_profile/published to the /freebase/type_status + instance named 'Published' + + + should also try to set the domain to some namespace that + has type:/type/domain + + """ + id = fb.absid(typeid) + + w = { + 'id': id, + '/freebase/type_profile/published': { + 'connect': 'insert', + 'type': '/freebase/type_status', + 'name': 'Published' + } + } + r = fb.mss.mqlwrite(w) + + print r['/freebase/type_profile/published']['connect'] |