A number of improvements from Arthur van Hoff:

 - Automatic saving of the mwLastWriteTime cookie
 - Add an option to specify Escaping (turned off by default)
 - Add a cursor-based iterator for enumerating large results sets
 - Add readmulti function for submitting multiple queries in one request
 - Add a search function
 - Support for default credentials, and escaping, permissions
 - Debug variable


git-svn-id: http://freebase-python.googlecode.com/svn/trunk@38 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2

1 files changed, 253 insertions, 26 deletions

diff --git a/metaweb-py/metaweb.py b/metaweb-py/metaweb.py
index d651332..e75ee0b 100644
--- a/metaweb-py/metaweb.py
+++ b/metaweb-py/metaweb.py
@@ -35,9 +35,12 @@
 # You can subscribe to the mailing list at http://lists.freebase.com/
 #
 
+import httplib
 import urllib        # URL encoding
 import urllib2       # Higher-level URL content fetching
 import simplejson    # JSON serialization and parsing
+import cookielib     # Cookie handling
+import os
 
 #
 # When experimenting, use the sandbox.freebase.com service.
@@ -50,7 +53,23 @@ readservice = '/api/service/mqlread'   # Path to mqlread service
 loginservice = '/api/account/login'     # Path to login service
 writeservice = '/api/service/mqlwrite'  # Path to mqlwrite service
 uploadservice = '/api/service/upload'   # Path to upload service
+searchservice = '/api/service/search'   # Path to search service
                  
+credentials = None      # default credential from login()
+escape = False          # default escape, set to 'html' for HTML escaping
+permission = None       # default permission used when creating new objects
+debug = False           # default debug setting
+
+# Install a CookieProcessor
+cookiefile = os.path.join(os.environ["HOME"], ".metaweb.cookies.txt")
+cookiejar = cookielib.LWPCookieJar()
+if os.path.isfile(cookiefile):
+    cookiejar.load(cookiefile)
+
+urllib2.install_opener(
+    urllib2.build_opener(
+        urllib2.HTTPCookieProcessor(cookiejar)))
+
 # If anything goes wrong when talking to a Metaweb service, we raise MQLError.
 class MQLError(Exception):
     def __init__(self, value):     # This is the exception constructor method
@@ -62,24 +81,35 @@ class MQLError(Exception):
 # If authentication credentials are supplied, use them in a cookie.
 # Raises MQLError if the query was invalid. Raises urllib2.HTTPError if
 # mqlread returns an HTTP status code other than 200 (which should not happen).
-def read(q, credentials=None):
+def read(q, credentials=credentials, escape=escape):
     # Put the query in an envelope
-    env = {'qname':{'query':q}}
-    # JSON serialize and URL encode the envelope and the query parameter
-    args = urllib.urlencode({'queries':simplejson.dumps(env)})
+    envelope = {'query':q}
+
+    # Add escape if needed
+    if escape != 'html':
+        envelope['escape'] = False if not escape else escape
+
+    # Encode the result
+    encoded = urllib.urlencode({'query': simplejson.dumps(envelope)})
+
     # Build the URL and create a Request object for it
-    url = 'http://%s%s?%s' % (host, readservice, args)
+    url = 'http://%s%s' % (host, readservice)
     req = urllib2.Request(url)
 
+    # The body of the POST request is encoded URL parameters
+    req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
     # Send our authentication credentials, if any, as a cookie.
     # The need for mqlread authentication is a temporary restriction.
-    if credentials: 
-        req.add_header('Cookie', credentials)
+    if credentials: req.add_header('Cookie', credentials)
+
+    # Use the encoded envelope as the value of the q parameter in the body
+    # of the request.  Specifying a body automatically makes this a POST.
+    req.add_data(encoded)
 
     # Now upen the URL and and parse its JSON content
     f = urllib2.urlopen(req)        # Open the URL
-    response = simplejson.load(f)   # Parse JSON response to an object
-    inner = response['qname']       # Open outer envelope; get inner envelope
+    inner = simplejson.load(f)      # Parse JSON response to an object
 
     # If anything was wrong with the invocation, mqlread will return an HTTP
     # error, and the code above with raise urllib2.HTTPError.
@@ -87,6 +117,10 @@ def read(q, credentials=None):
     # will get an error status code in the response envelope.  In this case
     # we raise our own MQLError exception.
     if not inner['code'].startswith('/api/status/ok'):
+        if debug: print q
+        if debug: print inner
+        if debug: print f.info()['X-Metaweb-Cost']
+        if debug: print f.info()['X-Metaweb-TID']
         error = inner['messages'][0]
         raise MQLError('%s: %s' % (error['code'], error['message']))
 
@@ -95,42 +129,53 @@ def read(q, credentials=None):
 
 # Submit the MQL query q and return the result as a Python object
 # This function behaves like read() above, but uses cursors so that
-# it works even for very large result sets
-def readall(q, credentials=None):
+# it works even for very large result sets. See also the cursor class below.
+def readall(q, credentials=credentials, escape=escape):
     # This is the start of the mqlread URL.
     # We just need to append the envelope to it
-    urlprefix = 'http://%s%s?queries=' % (host, readservice)
+    url = 'http://%s%s' % (host, readservice)
 
     # The query and most of the envelope are constant. We just need to append
     # the encoded cursor value and some closing braces to this prefix string
-    jsonq = simplejson.dumps(q);
-    envelopeprefix = urllib.quote_plus('{"q0":{"query":'+jsonq+',"cursor":')
-    
+    jsonq = simplejson.dumps(q)
+
+    # Add escape if needed
+    if escape != 'html':
+        jsonq += ',"escape":' + ('false' if not escape else escape)
+ 
     cursor = 'true'   # This is the initial value of the cursor
     results = []      # We accumulate results in this array
 
     # Loop until mqlread tells us there are no more results
     while cursor:
         # append the cursor and the closing braces to the envelope
-        envelope = envelopeprefix + urllib.quote_plus(cursor + '}}')
-        # append the envelope to the URL
-        url = urlprefix + envelope
+        envelope = urllib.urlencode({'query': '{"query":' + jsonq + ',"cursor":' + cursor + '}'})
 
         # Begin an HTTP request for the URL
         req = urllib2.Request(url)
 
+        # The body of the POST request is encoded URL parameters
+        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
         # Send our authentication credentials, if any, as a cookie.
         # The need for mqlread authentication is a temporary restriction.
         if credentials: 
             req.add_header('Cookie', credentials)
 
+        # Use the encoded envelope as the value of the q parameter in the body
+        # of the request.  Specifying a body automatically makes this a POST.
+        req.add_data(envelope)
+
         # Read and parse the URL contents
         f = urllib2.urlopen(req)          # Open URL
-        response = simplejson.load(f)     # Parse JSON response
-        inner = response['q0']            # Get inner envelope from outer
+        inner = simplejson.load(f)        # Parse JSON response
 
         # Raise a MQLError if there were errors
         if not inner['code'].startswith('/api/status/ok'):
+            if debug: print q
+            if debug: print inner
+            if debug: print f.info()['X-Metaweb-Cost']
+            if debug: print f.info()['X-Metaweb-TID']
             error = inner['messages'][0]
             raise MQLError('%s: %s' % (error['code'], error['message']))
 
@@ -145,6 +190,73 @@ def readall(q, credentials=None):
     # Now that we're done with the loop, return the results array
     return results
 
+# Submit multiple MQL queries and return the result as a Python array.
+# If authentication credentials are supplied, use them in a cookie.
+# Raises MQLError if the query was invalid. Raises urllib2.HTTPError if
+# mqlread returns an HTTP status code other than 200 (which should not happen).
+def readmulti(queries, credentials=credentials, escape=escape):
+    encoded = ""
+    for i in range(0, len(queries)):
+        # Put the query in an envelope
+        envelope = {'query':queries[i]}
+        # Add escape if needed
+        if escape != 'html':
+            envelope['escape'] = False if not escape else escape
+        if i > 0:
+            encoded += ","
+        encoded += '"q%d":%s' % (i, simplejson.dumps(envelope))
+
+    # URL encode the outer envelope
+    encoded = urllib.urlencode({'queries': "{" + encoded + "}"})
+
+    # Build the URL and create a Request object for it
+    url = 'http://%s%s' % (host, readservice)
+    req = urllib2.Request(url)
+
+    # The body of the POST request is encoded URL parameters
+    req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
+    # Send our authentication credentials, if any, as a cookie.
+    # The need for mqlread authentication is a temporary restriction.
+    if credentials: req.add_header('Cookie', credentials)
+
+    # Use the encoded envelope as the value of the q parameter in the body
+    # of the request.  Specifying a body automatically makes this a POST.
+    req.add_data(encoded)
+
+    # Now upen the URL and and parse its JSON content
+    f = urllib2.urlopen(req)        # Open the URL
+    inner = simplejson.load(f)      # Parse JSON response to an object
+
+    # If anything was wrong with the invocation, mqlread will return an HTTP
+    # error, and the code above with raise urllib2.HTTPError.
+    # If anything was wrong with the query, we won't get an HTTP error, but
+    # will get an error status code in the response envelope.  In this case
+    # we raise our own MQLError exception.
+    if not inner['code'].startswith('/api/status/ok'):
+        if debug: print queries
+        if debug: print inner
+        if debug: print f.info()['X-Metaweb-Cost']
+        if debug: print f.info()['X-Metaweb-TID']
+        error = inner['messages'][0]
+        raise MQLError('%s: %s' % (error['code'], error['message']))
+
+    # extract the results
+    results = []
+    for i in range(0, len(queries)):
+        result = inner["q%d" % i]
+        if not result['code'].startswith('/api/status/ok'):
+            if debug: print queries[i]
+            if debug: print result
+            if debug: print f.info()['X-Metaweb-Cost']
+            if debug: print f.info()['X-Metaweb-TID']
+            error = result['messages'][0]
+            raise MQLError('%s: %s' % (error['code'], error['message']))
+        results.append(result['result'])
+    
+    # If there was no error, then just return the result from the envelope
+    return results
+
 # Submit the specified username and password to the Metaweb login service.
 # Return opaque authentication credentials on success. 
 # Raise MQLError on failure.
@@ -167,6 +279,9 @@ def login(username, password):
         # Parse response body and raise a MQLError if login failed
         body = simplejson.loads(response.read())
         if not body['code'].startswith('/api/status/ok'):
+            if debug: print inner
+            if debug: print f.info()['X-Metaweb-Cost']
+            if debug: print f.info()['X-Metaweb-TID']
             error = body['messages'][0]
             raise MQLError('%s: %s' % (error['code'], error['message']))
 
@@ -187,20 +302,26 @@ def login(username, password):
 # Authentication credentials are required, obtained from login()
 # Raises MQLError if the query was invalid. Raises urllib2.HTTPError if
 # mqlwrite returns an HTTP status code other than 200
-def write(query, credentials):
+def write(query, credentials=credentials, escape=escape, permission=permission):
     # We're requesting this URL
     req = urllib2.Request('http://%s%s' % (host, writeservice))
     # Send our authentication credentials as a cookie
-    req.add_header('Cookie', credentials)
+    if credentials:
+        req.add_header('Cookie', credentials)
     # This custom header is required and guards against XSS attacks
     req.add_header('X-Metaweb-Request', 'True')
     # The body of the POST request is encoded URL parameters
     req.add_header('Content-type', 'application/x-www-form-urlencoded')
     # Wrap the query object in a query envelope
     envelope = {'qname': {'query': query}}
+    # Add escape if needed
+    if escape != 'html':
+        envelope['qname']['escape'] = (False if not escape else escape)
+    # Add permissions if needed
+    if permission:
+        envelope['qname']['use_permission_of'] = permission
     # JSON encode the envelope
     encoded = simplejson.dumps(envelope)
-    print encoded
     # Use the encoded envelope as the value of the q parameter in the body
     # of the request.  Specifying a body automatically makes this a POST.
     req.add_data(urllib.urlencode({'queries':encoded}))
@@ -208,7 +329,6 @@ def write(query, credentials):
     # Now do the POST
     f = urllib2.urlopen(req)
     response = simplejson.load(f)   # Parse HTTP response as JSON
-    print response
     inner = response['qname']       # Open outer envelope; get inner envelope
 
     # If anything was wrong with the invocation, mqlwrite will return an HTTP
@@ -217,26 +337,133 @@ def write(query, credentials):
     # in the response envelope.
     # we raise our own MQLError exception.
     if not inner['code'].startswith('/api/status/ok'):
+        if debug: print query
+        if debug: print inner
+        if debug: print f.info()['X-Metaweb-Cost']
+        if debug: print f.info()['X-Metaweb-TID']
         error = inner['messages'][0]
         raise MQLError('%s: %s' % (error['code'], error['message']))
 
+    # save cookie
+    cookiejar.save(cookiefile)
+
     # If there was no error, then just return the result from the envelope
     return inner['result']
 
 # Upload the specified content (and give it the specified type).
 # Return the guid of the /type/content object that represents it.
 # The returned guid can be used to retrieve the content with /api/trans/raw.
-def upload(content, type, credentials):
+def upload(content, type, credentials=credentials):
     # This is the URL we POST content to
     url = 'http://%s%s'%(host,uploadservice)
     # Build the HTTP request
     req = urllib2.Request(url, content)         # URL and content to POST
     req.add_header('Content-Type', type)        # Content type header
-    req.add_header('Cookie', credentials)       # Authentication header
+    if credentials:
+        req.add_header('Cookie', credentials)       # Authentication header
     req.add_header('X-Metaweb-Request', 'True') # Guard against XSS attacks
     f = urllib2.urlopen(req)                # POST the request
     response = simplejson.load(f)           # Parse the response
     if not response['code'].startswith('/api/status/ok'):
+        if debug: print inner
+        if debug: print f.info()['X-Metaweb-Cost']
+        if debug: print f.info()['X-Metaweb-TID']
         error = response['messages'][0]
         raise MQLError('%s: %s' % (error['code'], error['message']))
     return response['result']['id']         # Extract and return content id
+
+# Search for topics
+def search(query, type=None, start=0, limit=0):
+    args = {"query": query}
+    if type:
+        args["type"] = type
+    if start > 0:
+        args["start"] = start
+    if limit > 0:
+        args["limit"] = limit
+    url = 'http://%s%s?%s'%(host, searchservice, urllib.urlencode(args))
+    f = urllib2.urlopen(url)
+    response = simplejson.load(f)           # Parse the response
+    if not response['code'].startswith('/api/status/ok'):
+        if debug: print query
+        if debug: print inner
+        if debug: print f.info()['X-Metaweb-Cost']
+        if debug: print f.info()['X-Metaweb-TID']
+        error = response['messages'][0]
+        raise MQLError('%s: %s' % (error['code'], error['message']))
+    return response['result']
+
+# Cursor for iterating over large data sets
+# For example:
+#    query = {"name": None, "type":"/type/media_type"}
+#    for row in metaweb.cursor([query]):
+#        print row
+class cursor:
+    def __init__(self, query, credentials=credentials, escape=escape):
+        self.query = query
+        self.credentials = credentials
+        self.index = 0
+        self.results = []
+        self.cursor = 'true'
+        self.url = 'http://%s%s' % (host, readservice)
+        self.jsonq = simplejson.dumps(self.query)
+        if escape != 'html':
+            self.jsonq += ',"escape":' + ('false' if not escape else escape)
+ 
+    def __iter__(self):
+        return self
+
+    def next(self):
+        # return the next value
+        if self.index < len(self.results):
+            result = self.results[self.index]
+            self.index = self.index + 1
+            return result
+
+        # check if there is more
+        if not self.cursor:
+            raise StopIteration
+        
+        # append the cursor and the closing braces to the envelope
+        envelope = urllib.urlencode({'query': '{"query":' + self.jsonq + ',"cursor":' + self.cursor + '}'})
+
+        # Begin an HTTP request for the URL
+        req = urllib2.Request(self.url)
+
+        # The body of the POST request is encoded URL parameters
+        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
+        # Send our authentication credentials, if any, as a cookie.
+        # The need for mqlread authentication is a temporary restriction.
+        if self.credentials: req.add_header('Cookie', self.credentials)
+
+        # Use the encoded envelope as the value of the q parameter in the body
+        # of the request.  Specifying a body automatically makes this a POST.
+        req.add_data(envelope)
+
+        # Read and parse the URL contents
+        f = urllib2.urlopen(req)          # Open URL
+        inner = simplejson.load(f)        # Parse JSON response
+
+        # Raise a MQLError if there were errors
+        if not inner['code'].startswith('/api/status/ok'):
+            if debug: print self.query
+            if debug: print inner
+            if debug: print f.info()['X-Metaweb-Cost']
+            if debug: print f.info()['X-Metaweb-TID']
+            error = inner['messages'][0]
+            raise MQLError('%s: %s' % (error['code'], error['message']))
+
+        # Remember the next cursor
+        self.cursor = inner['cursor']
+        if self.cursor:                             # If it is not false, put it
+            self.cursor = '"' + self.cursor + '"'   #  in quotes as a JSON string
+
+        # Append this batch of results to the main array of results.
+        self.results = inner['result']
+        if len(self.results) == 0:
+            raise StopIteration
+
+        # Return the first result
+        self.index = 1
+        return self.results[0]