path: root/freebase/fcl/commands.py

# ==================================================================
# Copyright (c) 2007,2008,2009 Metaweb Technologies, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
# 
# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB
# TECHNOLOGIES OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ====================================================================

import os, sys, re, time
from fbutil import *
from cmdutil import *

try:
    import simplejson as json
except ImportError:
    import json

import freebase.rison as rison
from freebase.api import attrdict

from freebase.schema import connect_object, disconnect_object

def cmd_help(fb, command=None):
    """get help on commands

    %prog help [cmd]
    """

    if command is not None:
        cmd = fb.commands[command]
        print '   %s: %s' % (cmd.name, cmd.shortdoc)
        print cmd.doc
        return

    print """
the interface to this tool is loosely modeled on the
"svn" command-line tool.  many commands work slightly
differently from their unix and svn equivalents - 
read the doc for the command first.

use "%s help <subcommand>" for help on a particular
subcommand.
""" % fb.progpath

    fb.oparser.print_help()

    print 'available subcommands:'
    cmds = sorted(fb.commands.keys())

    for fk in cmds:
        cmd = fb.commands[fk]
        print '   %s: %s' % (cmd.name, cmd.shortdoc)


def cmd_wikihelp(fb):
    """get help on commands in mediawiki markup format

    %prog wikihelp
    """

    print """usage: %s subcommand [ args ... ]

    the interface to this tool is loosely modeled on the
    "svn" command-line tool.  many commands work slightly
    differently from their unix and svn equivalents - 
    read the doc for the command first.
    """ % fb.progpath

    print 'subcommands:'
    for fk in sorted(fb.commands.keys()):
        cmd = fb.commands[fk]

        # mediawiki markup cares about the difference between
        #  a blank line and a blank line with a bunch of
        #  spaces at the start of it.  ewww.
        doc = re.sub(r'\r?\n {0,8}', '\n        ', cmd.doc)

        print '==== %s %s: %s ====' % (fb.progpath, cmd.name, cmd.shortdoc)
        print doc
        print


# this command is disabled because it relies on some svn glue that
# was not sufficiently well thought out.
def old_cmd_pwid(fb):
    """show the "current working namespace id"

    %prog pwid

    by default, relative ids can't be resolved.  however, if you
    run the fb tool from a directory that has the svn property
    'freebase:id' set, relative ids will be resolved relative
    to that id.  the idea is that you can create an svn tree
    that parallels the freebase namespace tree.  

    for example:
        $ %prog pwid

        $ svn propset freebase:id "/freebase" .

        $ %prog pwid
        /freebase

    """
    if fb.cwid is not None:
        print fb.cwid
    else:
        print ''


@option('long', '-l', default=False, action='store_true', help='show time and permission')
@option('timesort', '-t', default=False, action='store_true', help='sort by time')
@option('revsort', '-r', default=False, action='store_true', help='reverse sort order')
def cmd_ls(fb, path=None, long=False, timesort=False, revsort=False):
    """list the keys in a namespace

    %prog ls [id]
    """
    path = fb.absid(path)
    # ignore trailing /
    if len(path) > 1:
        path = re.sub('/$', '', path)

    kq = {
        'value': None,
        'namespace': {
            'id': None
        },
        'sort': 'value',
        'optional':True
     }

    if long or timesort:
        kq['link'] = {
            'timestamp': None,
            'creator': None
          }
    if timesort:
        kq['sort'] = 'link.timestamp'

    if revsort:
        kq['sort'] = '-' + kq['sort']

    q = {'id': path,
         '/type/namespace/keys': [kq]
         }

    r = fb.mss.mqlread(q)
    if r is None:
        raise CmdException('query for id %r failed' % path)

    #sys.stdout.write(' '.join([mk.value for mk in r['/type/namespace/keys']]))

    for mk in r['/type/namespace/keys']:
        if long:
            # trim off the /user/
            creator = re.sub('^/user/', '', mk.link.creator)
            timestamp = mk.link.timestamp
            fb.out(mk.value, creator, timestamp)
        else:
            fb.out(mk.value)

    if 0:
        suffix = ''
        if ('/type/namespace' in mk.namespace.type
            or '/type/domain' in mk.namespace.type):
            suffix = '/'
        print mk.value, mk.namespace.id+suffix


@complete('path')
def cmd_mkdir(fb, path):
    """create a new freebase namespace
    %prog mkdir id

    create a new instance of /type/namespace at the given
    point in id space.  if id already exists, it should be
    a namespace.
    """
    path = fb.absid(path)
    # ignore trailing /
    path = re.sub('/$', '', path)
    dir,file = dirsplit(path)
    wq = { 'create': 'unless_exists',
           'key':{
               'connect': 'insert',
               'namespace': dir,
               'value': file
               },
           'name': path,
           'type': '/type/namespace'
         }

    r = fb.mss.mqlwrite(wq)

@complete('path', 'path')
def cmd_ln(fb, src, dst):
    """create a namespace key
    %prog ln srcid dstid

    create a new namespace link at dstid to the object
    currently at srcid.
    """
    src = fb.absid(src)
    dst = fb.absid(dst)
    
    # ignore trailing /
    src = re.sub('/$', '', src)
    dst = re.sub('/$', '', dst)

    return connect_object(fb.mss, src, dst)

def cmd_rm(fb, path):
    """unlink a namespace key
    %prog rm id
    
    remove the /type/key that connects the given id to its
    parent.  id must be a path for this to make any sense.

    note that is like unix 'unlink' rather than 'rm'.
    it won't complain if the 'subdirectory' contains data,
    since that data will still be accessible to other queries.
    it's not like 'rm -rf' either, because it doesn't
    disturb anything other than the one directory entry.
    """
    path = fb.absid(path)
    # ignore trailing /
    path = re.sub('/$', '', path)

    return disconnect_object(fb.mss, path)

@complete('path', 'path')
def cmd_mv(fb, src, dst):
    """rename srcid to dstid.
    %prog mv srcid dstid

    equivalent to:
    $ fb ln <srcid> <dstid>
    $ fb rm <srcid>
    """
    cmd_ln(fb, src, dst)
    cmd_rm(fb, src)

def cmd_cd(fb, path):
    path = fb.absid(path)
    fb.pwd = path

def cmd_pwd(fb):
    print fb.pwd

def cmd_cat(fb, id, include_headers=False):
    """download a document from freebase to stdout
    %prog cat id

    equivalent to "%prog get id -".
    """
    return cmd_get(fb, id, localfile='-', include_headers=include_headers)

def cmd_shell(fb):
    import readline
    import shlex
    cache = {}
    def complete_cmd(text, state):
        def complete_path(t, i):
            if t == '':
                pwd = fb.pwd
                k = 'value'
                v = None
            elif t.startswith('/'):
                p = t.split('/')
                k = 'value~='
                v = '^'+p.pop()+'.*'
                pwd = '/'.join(p)
            else:
                pwd = fb.pwd
                k = 'value~='
                v = '^'+t+'.*'

            ckey = '%s:%s' % (k,v)
            if ckey in cache:
                r = cache[ckey]
            else:
                q = {'id': pwd,
                     '/type/namespace/keys': [{k:v, 'value':None}]
                     }
                r = fb.mss.mqlread(q)
                cache[ckey] = r

            return r['/type/namespace/keys'][i]['value']

        completes = {'path':complete_path}
        lb = readline.get_line_buffer()
        if lb == text:
            m = [ c for c in fb.commands if c.startswith(text) ]
            return m[state]
        else:
            args = lb.split()
            cmd = args.pop(0)
            la = len(args)-1
            if la == -1:
                la = 0

            types = fb.commands[cmd].func.types
            t = types[la]
            return completes.get(t, lambda x,y: None)(text, state)


    readline.set_completer(complete_cmd)
    readline.parse_and_bind('tab: complete')
    graphs = {
        'http://trunk.qa.metaweb.com':'qa',
        'http://branch.qa.metaweb.com':'qa',
        'http://www.freebase.com':'otg',
        'http://api.freebase.com':'otg',
        'http://freebase.com':'otg',
        'http://www.sandbox-freebase.com':'sandbox',
        'http://api.sandbox-freebase.com':'sandbox',
        'http://sandbox-freebase.com':'sandbox'
    }

    gname = graphs.get(fb.mss.service_url, 'unknown')
    while True:
        try:
            line = raw_input((fb.mss.username+'@' if fb.mss.username else '') +gname+': '+fb.pwd+' > ')
        except EOFError:
            print
            break

        if line:
            args = shlex.split(line)
            cmd = args.pop(0)
            if cmd in fb.commands:
                fb.dispatch(fb.commands[cmd], args, {});
            else:
                print >>sys.stderr, "No such command: "+cmd
        
def cmd_get(fb, id, localfile=None, include_headers=False):
    """download a file from freebase
    %prog get id [localfile]

    download the document or image with the given id from freebase
    into localfile.  localfile '-' means stdout.  localfile
    defaults to a file in the current directory with the same name
    as the last key in the path, possibly followed by a metadata
    extension like .html or .txt.
    """
    id = fb.absid(id)
    dir,file = dirsplit_unsafe(id)

    def read_content(id, content_only=False):
        c = attrdict(id=id)
        cq = { 'id': id,
               'type': [],
               '/common/document/content': None,
               '/common/document/source_uri': None,
               '/type/content/media_type': { 'name':None,
                                             'optional': True },
               #'/type/content/text_encoding': { 'name':None },
               '/type/content/blob_id':None,
             }
        cd = fb.mss.mqlread(cq)
        if cd is None:
            raise CmdException('no match for id %r' % id)
        
        if '/type/content' in cd.type:
            c.media_type = cd['/type/content/media_type'].name
            #c.text_encoding = cd['/type/content/text_encoding'].name
            c.sha256 = cd['/type/content/blob_id']
            return c

        if content_only:
            raise CmdException('%s is not a content id' % id)

        cid = cd['/common/document/content']
        if cid is not None:
            return read_content(cid, content_only=True)

        # in this case we don't have a content object
        if cd['/common/document/source_uri'] is not None:
            return None

        raise CmdException('%s is not a content or document id' % id)


    content = read_content(id)
    log.debug('COBJ %r' % content)
    if content is not None:
        fileext = media_type_to_extension.get(content.media_type, None)
    else:
        fileext = None

    if localfile == '-':
        ofp = sys.stdout
    else:
        if localfile is None:
            implicit_outfile = True
            localfile = file
        elif re.match(r'[/\\]$', localfile):
            implicit_outfile = True
            localfile = localfile + file
        else:
            implicit_outfile = False
        localfile = os.path.abspath(localfile)

        # add file extension based on content-type:
        #  should be an option to disable this
        if implicit_outfile and fileext is not None:
            localfile += '.' + fileext

        # if we didn't explicitly name the output file,
        #  don't destroy an existing file
        localfile_base = localfile
        count = 0
        while implicit_outfile and os.path.exists(localfile):
            count += 1
            localfile = '%s.%d' % (localfile_base, count)
        ofp = open(localfile, 'wb')

    body = fb.mss.trans(id)

    if include_headers:
        # XXX show content-type, what else?
        pass

    ofp.write(body)

    if localfile != '-':
        print ('%s saved (%d bytes)' % (localfile, len(body)))
        ofp.close()


def cmd_put(fb, localfile, id=None, content_type=None):
    """upload a document to freebase   -- EXPERIMENTAL
    %prog put localfile [id] [content-type]

    upload the document or image in localfile to given freebase
    id.  if localfile is '-' the data will be read from stdin.

    if id is missing or empty, a new document will be created.
    later the id might default to something computed from localfile
    and any svn attributes it has.

    output: a single line, the id of the document.
    """
    if content_type is None:
        ext = re.sub('^.*\.([^/.]+)$', r'\1', localfile)
        media_type = extension_to_media_type.get(ext, None)

        if media_type is None:
            raise CmdException('could not infer a media type from extension %r: please specify it'
                               % ext)

        if media_type.startswith('text/'):
            # this is a bad assumption.  should sniff it?
            text_encoding = 'utf-8'
            content_type = '%s;charset=%s' % (media_type, text_encoding)
        else:
            content_type = media_type

    new_id = None
    if id is not None:
        idinfo = fb.mss.mqlread({ 'id': id, 'type': '/common/document' })
        if idinfo is None:
            new_id = id
            id = None

    body = open(localfile, 'rb').read()
    r = fb.mss.upload(body, content_type, document_id=id)

    if new_id is None:
        print r.document
    else:
        cmd_ln(fb, r.document, new_id)
        print new_id


def cmd_dump(fb, id):
    """show all properties of a freebase object
    %prog dump object_id
    """
    id = fb.absid(id)

    import inspection

    r = inspection.inspect_object(fb.mss, id)
    if r is None:
        raise CmdException('no match for id %r' % id)

    for k in sorted(r.keys()):
        vs = r[k]
        for v in vs:
            id = v.get('id', '')

            name = '%r' % (v.get('name') or v.get('value'))
            if name == 'None': name = ''

            type = v.get('type', '')
            if type == '/type/text':
                extra = v.get('lang', '')
            elif type == '/type/key':
                extra = v.get('namespace', '')
            else:
                extra = ''

            fb.out(k, id, name, type, extra)
                    

def cmd_pget(fb, id, propid):
    """get a property of a freebase object  -- EXPERIMENTAL
    %prog pget object_id property_id

    get the property named by property_id from the object.

    XXX output quoting is not well specified.

    property_id must be a fully qualified id for now.

    prints one line for each match.

    if propid ends in '*' this does a wildcard for a particular type.
    """
    id = fb.absid(id)
    proptype, propkey = dirsplit(propid)

    if propkey != '*':
        # look up the prop
        q = { 'id': id,
               propid: [{}],
             }
        r = fb.mss.mqlread(q)
        for v in r[propid]:
            if 'value' in v:
                print v.value
            else:
                print v.id

    else:
        # look up the prop
        q = { 'id': id,
               '*': [{}],
             }
        if isinstance(proptype, basestring):
            q['type'] = proptype

        r = fb.mss.mqlread(q)

        for k in sorted(r.keys()):
            v = r[k];
            if 'value' in v:
                print '%s %s' % (k, v.value)
            else:
                print '%s %s' % (k, v.id)

def cmd_pdel(fb, id, propid, oldval):
    """delete a property of a freebase object   -- EXPERIMENTAL
    %prog pdel object_id property_id oldvalue

    set the property named by property_id on the object.
    value is an id or a json value.  XXX this is ambiguous.

    property_id must be a fully qualified id for now.

    for now you need to provide a "oldval" argument,
    later this tool will query and perhaps prompt if the
    deletion is ambiguous.

    prints a single line, either 'deleted' or 'missing'
    """
    return cmd_pset(fb, id, propid, None, oldval)


def cmd_touch(fb):
    """bypass any cached query results the service may have.  use sparingly.
    """
    fb.mss.mqlflush()


@complete('path', 'path', None)
def cmd_pset(fb, id, propkey, val, oldval=None, extra=None):
    """set a property of a freebase object  -- EXPERIMENTAL
    %prog pset object_id property_id value

    set the property named by property_id on the object.
    value is an id or a json value.  XXX this is ambiguous.

    property_id must be a fully qualified id for now.

    if the property should be a unique property, this will
    write with 'connect:update'.  if the property may have
    multiple, it is written with 'connect:insert'.

    prints a single line, either 'inserted' or 'present'
    """
    id = fb.absid(id)

    propid = fb.absprop(propkey)

    # look up the prop
    pq = { 'id': propid,
           'type': '/type/property',
           'name': None,
           'unique': None,
           'expected_type': {
               'id':   None,
               'name': None,
               'default_property': None,
               'optional': True,
           },
         }

    prop = fb.mss.mqlread(pq)

    if prop is None:
        raise CmdException('can\'t resolve property key %r - use an absolute id' % propid);

    if propid.startswith('/type/object/') or propid.startswith('/type/value/'):
        propkey = re.sub('/type/[^/]+/', '', propid);
    else:
        propkey = propid

    wq = { 'id': id,
           propkey: {
           }
         }

    if val is None:
        val = oldval
        wq[propkey]['connect'] = 'delete'
    elif prop.unique:
        wq[propkey]['connect'] = 'update'
    else:
        wq[propkey]['connect'] = 'insert'

    if prop.expected_type is None:
        wq[propkey]['id'] = val
    elif prop.expected_type.id not in value_types:
        wq[propkey]['id'] = val
    elif prop.expected_type.id == '/type/float':
        wq[propkey]['value'] = float(val)
    elif prop.expected_type.id == '/type/int':
        wq[propkey]['value'] = int(val)
    elif prop.expected_type.id == '/type/boolean':
        wq[propkey]['value'] = bool(val)
    elif prop.expected_type.id == '/type/text':
        wq[propkey]['value'] = val
        if extra is not None:
            lang = extra
        else:
            lang = '/lang/en'
        wq[propkey]['lang'] = lang
    elif prop.expected_type.id == '/type/key':
        wq[propkey]['value'] = val
        if extra is not None:
            wq[propkey]['namespace'] = extra
        else:
            raise CmdException('must specify a namespace to pset /type/key')
    else:
        wq[propkey]['value'] = val


    r = fb.mss.mqlwrite(wq)
    print r[propkey]['connect']
    
def cmd_login(fb, username=None, password=None):
    """login to the freebase service
    %prog login [username [password]]

    cookies are maintained in a file so
    they are available to the next invocation.
    prompts for username and password if not given
    """
    import getpass
    if username is None:
        sys.stdout.write('freebase.com username: ')
        username = sys.stdin.readline()
        if not username:
            raise CmdException('usernmae required for login')
        username = re.compile('\n$').sub('', username)
    if password is None:
        password = getpass.getpass('freebase.com password: ')

    fb.mss.username = username
    fb.mss.password = password
    fb.mss.login()

def cmd_logout(fb):
    """logout from the freebase service
    %prog logout

    deletes the login cookies
    """
    fb.cookiejar.clear(domain=fb.service_host.split(':')[0])


def cmd_find(fb, qstr):
    """print all ids matching a given constraint.

    if the query string starts with "{" it is treated as json.
    otherwise it is treated as o-rison.

    %prog find
    """
    if qstr.startswith('{'):
        q = json.loads(qstr)
    else:
        q = rison.loads('(' + qstr + ')')

    if 'id' not in q:
        q['id'] = None

    results = fb.mss.mqlreaditer(q)
    for r in results:
        fb.out(r.id)


def cmd_q(fb, qstr):
    """run a freebase query.

    if the query string starts with "{" it is treated as json.
    otherwise it is treated as o-rison.

    dump the result as json.
    
    %prog q
    """
    if qstr.startswith('{'):
        q = json.loads(qstr)
    else:
        q = rison.loads('(' + qstr + ')')

    # done this way for streaming
    first = True
    for result in fb.mss.mqlreaditer(q):
        if first:
            first = False
            print '[',
        else:
            print ',',
        print json.dumps(result, indent=2),
    print ']'

def cmd_open(fb, id):
    """open a web browser on the given id.  works on OSX only for now.
    
    %prog open /some/id
    """
    path = fb.absid(id)
    os.system("open 'http://%s/view%s'" % (fb.service_host, path))


def cmd_log(fb, id):
    """log changes pertaining to a given id.

    INCOMPLETE
    
    %prog log /some/id
    """

    null = None
    true = True
    false = False

    baseq = {
            'type': '/type/link',
            'source': null,
            'master_property': null,
            'attribution': null,
            'timestamp': null,
            'operation': null,
            'valid': null,
            'sort': '-timestamp'
        };
    
    queries = [
        {
        'target_value': { '*': null },
        'target': { 'id': null, 'name': null, 'optional': true },
        },
        {
        'target': { 'id': null, 'name': null },
        }]

    for i,q in list(enumerate(queries)):
        q.update(baseq)
        queries[i] = [q]


    valuesfrom,linksfrom = fb.mss.mqlreadmulti(queries)

    for link in linksfrom:
        # fb.trow(link.master_property.id, ...)
        print json.dumps(link, indent=2)

    for link in valuesfrom:
        # fb.trow(link.master_property.id, ...)
        print json.dumps(link, indent=2)

    

def cmd_search(fb, what):
    """Search freebase for "query" and print out 10 matching ids
    
    %prog search "some query" 
    """
        
    r = fb.mss.search(what, format="ids", limit=10)
    for id in r:    
        print id