diff options
Diffstat (limited to 'metaweb-py')
| -rw-r--r-- | metaweb-py/metaweb.py | 469 |
1 files changed, 0 insertions, 469 deletions
diff --git a/metaweb-py/metaweb.py b/metaweb-py/metaweb.py deleted file mode 100644 index e75ee0b..0000000 --- a/metaweb-py/metaweb.py +++ /dev/null @@ -1,469 +0,0 @@ -#======================================================================== -# Copyright (c) 2007, Metaweb Technologies, Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METAWEB TECHNOLOGIES BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ======================================================================== -# -# This is the full "metaweb.py" module from the Metaweb API documentation -# -# In the documentation, each function is presented as a separate -# example. This is the whole file. -# -# If you find any errors or have suggestions for improving this module, -# send them to the Freebase developers mailing list: developers@freebase.com -# You can subscribe to the mailing list at http://lists.freebase.com/ -# - -import httplib -import urllib # URL encoding -import urllib2 # Higher-level URL content fetching -import simplejson # JSON serialization and parsing -import cookielib # Cookie handling -import os - -# -# When experimenting, use the sandbox.freebase.com service. -# Every Monday, sandbox.freebase.com is erased and it is updated -# with a fresh copy of data from www.freebase.com. This makes -# it an ideal place to experiment. -# -host = 'sandbox.freebase.com' # The Metaweb host -readservice = '/api/service/mqlread' # Path to mqlread service -loginservice = '/api/account/login' # Path to login service -writeservice = '/api/service/mqlwrite' # Path to mqlwrite service -uploadservice = '/api/service/upload' # Path to upload service -searchservice = '/api/service/search' # Path to search service - -credentials = None # default credential from login() -escape = False # default escape, set to 'html' for HTML escaping -permission = None # default permission used when creating new objects -debug = False # default debug setting - -# Install a CookieProcessor -cookiefile = os.path.join(os.environ["HOME"], ".metaweb.cookies.txt") -cookiejar = cookielib.LWPCookieJar() -if os.path.isfile(cookiefile): - cookiejar.load(cookiefile) - -urllib2.install_opener( - urllib2.build_opener( - urllib2.HTTPCookieProcessor(cookiejar))) - -# If anything goes wrong when talking to a Metaweb service, we raise MQLError. -class MQLError(Exception): - def __init__(self, value): # This is the exception constructor method - self.value = value - def __str__(self): # Convert error object to a string - return repr(self.value) - -# Submit the MQL query q and return the result as a Python object. -# If authentication credentials are supplied, use them in a cookie. -# Raises MQLError if the query was invalid. Raises urllib2.HTTPError if -# mqlread returns an HTTP status code other than 200 (which should not happen). -def read(q, credentials=credentials, escape=escape): - # Put the query in an envelope - envelope = {'query':q} - - # Add escape if needed - if escape != 'html': - envelope['escape'] = False if not escape else escape - - # Encode the result - encoded = urllib.urlencode({'query': simplejson.dumps(envelope)}) - - # Build the URL and create a Request object for it - url = 'http://%s%s' % (host, readservice) - req = urllib2.Request(url) - - # The body of the POST request is encoded URL parameters - req.add_header('Content-type', 'application/x-www-form-urlencoded') - - # Send our authentication credentials, if any, as a cookie. - # The need for mqlread authentication is a temporary restriction. - if credentials: req.add_header('Cookie', credentials) - - # Use the encoded envelope as the value of the q parameter in the body - # of the request. Specifying a body automatically makes this a POST. - req.add_data(encoded) - - # Now upen the URL and and parse its JSON content - f = urllib2.urlopen(req) # Open the URL - inner = simplejson.load(f) # Parse JSON response to an object - - # If anything was wrong with the invocation, mqlread will return an HTTP - # error, and the code above with raise urllib2.HTTPError. - # If anything was wrong with the query, we won't get an HTTP error, but - # will get an error status code in the response envelope. In this case - # we raise our own MQLError exception. - if not inner['code'].startswith('/api/status/ok'): - if debug: print q - if debug: print inner - if debug: print f.info()['X-Metaweb-Cost'] - if debug: print f.info()['X-Metaweb-TID'] - error = inner['messages'][0] - raise MQLError('%s: %s' % (error['code'], error['message'])) - - # If there was no error, then just return the result from the envelope - return inner['result'] - -# Submit the MQL query q and return the result as a Python object -# This function behaves like read() above, but uses cursors so that -# it works even for very large result sets. See also the cursor class below. -def readall(q, credentials=credentials, escape=escape): - # This is the start of the mqlread URL. - # We just need to append the envelope to it - url = 'http://%s%s' % (host, readservice) - - # The query and most of the envelope are constant. We just need to append - # the encoded cursor value and some closing braces to this prefix string - jsonq = simplejson.dumps(q) - - # Add escape if needed - if escape != 'html': - jsonq += ',"escape":' + ('false' if not escape else escape) - - cursor = 'true' # This is the initial value of the cursor - results = [] # We accumulate results in this array - - # Loop until mqlread tells us there are no more results - while cursor: - # append the cursor and the closing braces to the envelope - envelope = urllib.urlencode({'query': '{"query":' + jsonq + ',"cursor":' + cursor + '}'}) - - # Begin an HTTP request for the URL - req = urllib2.Request(url) - - # The body of the POST request is encoded URL parameters - req.add_header('Content-type', 'application/x-www-form-urlencoded') - - # Send our authentication credentials, if any, as a cookie. - # The need for mqlread authentication is a temporary restriction. - if credentials: - req.add_header('Cookie', credentials) - - # Use the encoded envelope as the value of the q parameter in the body - # of the request. Specifying a body automatically makes this a POST. - req.add_data(envelope) - - # Read and parse the URL contents - f = urllib2.urlopen(req) # Open URL - inner = simplejson.load(f) # Parse JSON response - - # Raise a MQLError if there were errors - if not inner['code'].startswith('/api/status/ok'): - if debug: print q - if debug: print inner - if debug: print f.info()['X-Metaweb-Cost'] - if debug: print f.info()['X-Metaweb-TID'] - error = inner['messages'][0] - raise MQLError('%s: %s' % (error['code'], error['message'])) - - # Append this batch of results to the main array of results. - results.extend(inner['result']) - - # Finally, get the new value of the cursor for the next iteration - cursor = inner['cursor'] - if cursor: # If it is not false, put it - cursor = '"' + cursor + '"' # in quotes as a JSON string - - # Now that we're done with the loop, return the results array - return results - -# Submit multiple MQL queries and return the result as a Python array. -# If authentication credentials are supplied, use them in a cookie. -# Raises MQLError if the query was invalid. Raises urllib2.HTTPError if -# mqlread returns an HTTP status code other than 200 (which should not happen). -def readmulti(queries, credentials=credentials, escape=escape): - encoded = "" - for i in range(0, len(queries)): - # Put the query in an envelope - envelope = {'query':queries[i]} - # Add escape if needed - if escape != 'html': - envelope['escape'] = False if not escape else escape - if i > 0: - encoded += "," - encoded += '"q%d":%s' % (i, simplejson.dumps(envelope)) - - # URL encode the outer envelope - encoded = urllib.urlencode({'queries': "{" + encoded + "}"}) - - # Build the URL and create a Request object for it - url = 'http://%s%s' % (host, readservice) - req = urllib2.Request(url) - - # The body of the POST request is encoded URL parameters - req.add_header('Content-type', 'application/x-www-form-urlencoded') - - # Send our authentication credentials, if any, as a cookie. - # The need for mqlread authentication is a temporary restriction. - if credentials: req.add_header('Cookie', credentials) - - # Use the encoded envelope as the value of the q parameter in the body - # of the request. Specifying a body automatically makes this a POST. - req.add_data(encoded) - - # Now upen the URL and and parse its JSON content - f = urllib2.urlopen(req) # Open the URL - inner = simplejson.load(f) # Parse JSON response to an object - - # If anything was wrong with the invocation, mqlread will return an HTTP - # error, and the code above with raise urllib2.HTTPError. - # If anything was wrong with the query, we won't get an HTTP error, but - # will get an error status code in the response envelope. In this case - # we raise our own MQLError exception. - if not inner['code'].startswith('/api/status/ok'): - if debug: print queries - if debug: print inner - if debug: print f.info()['X-Metaweb-Cost'] - if debug: print f.info()['X-Metaweb-TID'] - error = inner['messages'][0] - raise MQLError('%s: %s' % (error['code'], error['message'])) - - # extract the results - results = [] - for i in range(0, len(queries)): - result = inner["q%d" % i] - if not result['code'].startswith('/api/status/ok'): - if debug: print queries[i] - if debug: print result - if debug: print f.info()['X-Metaweb-Cost'] - if debug: print f.info()['X-Metaweb-TID'] - error = result['messages'][0] - raise MQLError('%s: %s' % (error['code'], error['message'])) - results.append(result['result']) - - # If there was no error, then just return the result from the envelope - return results - -# Submit the specified username and password to the Metaweb login service. -# Return opaque authentication credentials on success. -# Raise MQLError on failure. -def login(username, password): - # Establish a connection to the server and make a request. - # Note that we use the low-level httplib library instead of urllib2. - # This allows us to manage cookies explicitly. - conn = httplib.HTTPConnection(host) - conn.request('POST', # POST the request - loginservice, # The URL path /api/account/login - # The body of the request: encoded username/password - urllib.urlencode({'username':username, 'password':password}), - # This header specifies how the body of the post is encoded. - {'Content-type': 'application/x-www-form-urlencoded'}) - - # Get the response from the server - response = conn.getresponse() - - if response.status == 200: # We get HTTP 200 OK even if login fails - # Parse response body and raise a MQLError if login failed - body = simplejson.loads(response.read()) - if not body['code'].startswith('/api/status/ok'): - if debug: print inner - if debug: print f.info()['X-Metaweb-Cost'] - if debug: print f.info()['X-Metaweb-TID'] - error = body['messages'][0] - raise MQLError('%s: %s' % (error['code'], error['message'])) - - # Otherwise return cookies to serve as authentication credentials. - # The set-cookie header holds one or more cookie specifications, - # separated by commas. Each specification is a name, an equal - # sign, a value, and one or more trailing clauses that consist - # of a semicolon and some metadata. We don't care about the - # metadata. We just want to return a comma-separated list of - # name=value pairs. - cookies = response.getheader('set-cookie').split(',') - return ';'.join([c[0:c.index(';')] for c in cookies]) - else: # This should never happen - raise MQLError('HTTP Error: %d %s' % (response.status,response.reason)) - - -# Submit the MQL write q and return the result as a Python object. -# Authentication credentials are required, obtained from login() -# Raises MQLError if the query was invalid. Raises urllib2.HTTPError if -# mqlwrite returns an HTTP status code other than 200 -def write(query, credentials=credentials, escape=escape, permission=permission): - # We're requesting this URL - req = urllib2.Request('http://%s%s' % (host, writeservice)) - # Send our authentication credentials as a cookie - if credentials: - req.add_header('Cookie', credentials) - # This custom header is required and guards against XSS attacks - req.add_header('X-Metaweb-Request', 'True') - # The body of the POST request is encoded URL parameters - req.add_header('Content-type', 'application/x-www-form-urlencoded') - # Wrap the query object in a query envelope - envelope = {'qname': {'query': query}} - # Add escape if needed - if escape != 'html': - envelope['qname']['escape'] = (False if not escape else escape) - # Add permissions if needed - if permission: - envelope['qname']['use_permission_of'] = permission - # JSON encode the envelope - encoded = simplejson.dumps(envelope) - # Use the encoded envelope as the value of the q parameter in the body - # of the request. Specifying a body automatically makes this a POST. - req.add_data(urllib.urlencode({'queries':encoded})) - - # Now do the POST - f = urllib2.urlopen(req) - response = simplejson.load(f) # Parse HTTP response as JSON - inner = response['qname'] # Open outer envelope; get inner envelope - - # If anything was wrong with the invocation, mqlwrite will return an HTTP - # error, and the code above with raise urllib2.HTTPError. - # If anything was wrong with the query, we will get an error status code - # in the response envelope. - # we raise our own MQLError exception. - if not inner['code'].startswith('/api/status/ok'): - if debug: print query - if debug: print inner - if debug: print f.info()['X-Metaweb-Cost'] - if debug: print f.info()['X-Metaweb-TID'] - error = inner['messages'][0] - raise MQLError('%s: %s' % (error['code'], error['message'])) - - # save cookie - cookiejar.save(cookiefile) - - # If there was no error, then just return the result from the envelope - return inner['result'] - -# Upload the specified content (and give it the specified type). -# Return the guid of the /type/content object that represents it. -# The returned guid can be used to retrieve the content with /api/trans/raw. -def upload(content, type, credentials=credentials): - # This is the URL we POST content to - url = 'http://%s%s'%(host,uploadservice) - # Build the HTTP request - req = urllib2.Request(url, content) # URL and content to POST - req.add_header('Content-Type', type) # Content type header - if credentials: - req.add_header('Cookie', credentials) # Authentication header - req.add_header('X-Metaweb-Request', 'True') # Guard against XSS attacks - f = urllib2.urlopen(req) # POST the request - response = simplejson.load(f) # Parse the response - if not response['code'].startswith('/api/status/ok'): - if debug: print inner - if debug: print f.info()['X-Metaweb-Cost'] - if debug: print f.info()['X-Metaweb-TID'] - error = response['messages'][0] - raise MQLError('%s: %s' % (error['code'], error['message'])) - return response['result']['id'] # Extract and return content id - -# Search for topics -def search(query, type=None, start=0, limit=0): - args = {"query": query} - if type: - args["type"] = type - if start > 0: - args["start"] = start - if limit > 0: - args["limit"] = limit - url = 'http://%s%s?%s'%(host, searchservice, urllib.urlencode(args)) - f = urllib2.urlopen(url) - response = simplejson.load(f) # Parse the response - if not response['code'].startswith('/api/status/ok'): - if debug: print query - if debug: print inner - if debug: print f.info()['X-Metaweb-Cost'] - if debug: print f.info()['X-Metaweb-TID'] - error = response['messages'][0] - raise MQLError('%s: %s' % (error['code'], error['message'])) - return response['result'] - -# Cursor for iterating over large data sets -# For example: -# query = {"name": None, "type":"/type/media_type"} -# for row in metaweb.cursor([query]): -# print row -class cursor: - def __init__(self, query, credentials=credentials, escape=escape): - self.query = query - self.credentials = credentials - self.index = 0 - self.results = [] - self.cursor = 'true' - self.url = 'http://%s%s' % (host, readservice) - self.jsonq = simplejson.dumps(self.query) - if escape != 'html': - self.jsonq += ',"escape":' + ('false' if not escape else escape) - - def __iter__(self): - return self - - def next(self): - # return the next value - if self.index < len(self.results): - result = self.results[self.index] - self.index = self.index + 1 - return result - - # check if there is more - if not self.cursor: - raise StopIteration - - # append the cursor and the closing braces to the envelope - envelope = urllib.urlencode({'query': '{"query":' + self.jsonq + ',"cursor":' + self.cursor + '}'}) - - # Begin an HTTP request for the URL - req = urllib2.Request(self.url) - - # The body of the POST request is encoded URL parameters - req.add_header('Content-type', 'application/x-www-form-urlencoded') - - # Send our authentication credentials, if any, as a cookie. - # The need for mqlread authentication is a temporary restriction. - if self.credentials: req.add_header('Cookie', self.credentials) - - # Use the encoded envelope as the value of the q parameter in the body - # of the request. Specifying a body automatically makes this a POST. - req.add_data(envelope) - - # Read and parse the URL contents - f = urllib2.urlopen(req) # Open URL - inner = simplejson.load(f) # Parse JSON response - - # Raise a MQLError if there were errors - if not inner['code'].startswith('/api/status/ok'): - if debug: print self.query - if debug: print inner - if debug: print f.info()['X-Metaweb-Cost'] - if debug: print f.info()['X-Metaweb-TID'] - error = inner['messages'][0] - raise MQLError('%s: %s' % (error['code'], error['message'])) - - # Remember the next cursor - self.cursor = inner['cursor'] - if self.cursor: # If it is not false, put it - self.cursor = '"' + self.cursor + '"' # in quotes as a JSON string - - # Append this batch of results to the main array of results. - self.results = inner['result'] - if len(self.results) == 0: - raise StopIteration - - # Return the first result - self.index = 1 - return self.results[0] |