make freebase.schema more readable, moved some tests, finalizing the stuff that should actually be in the package

git-svn-id: http://freebase-python.googlecode.com/svn/trunk@178 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2

8 files changed, 403 insertions, 1105 deletions

diff --git a/freebase/api/session.py b/freebase/api/session.py
index d717d53..ac3b5e4 100644
--- a/freebase/api/session.py
+++ b/freebase/api/session.py
@@ -39,7 +39,7 @@ declarations for external metaweb api.
 
 
 __all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict']
-__version__ = '0.5'
+__version__ = '1.0'
 
 import os, sys, re
 import cookielib
diff --git a/freebase/schema.py b/freebase/schema.py
index 62e5389..459c51a 100644
--- a/freebase/schema.py
+++ b/freebase/schema.py
@@ -2,6 +2,44 @@
 from freebase.api.session import HTTPMetawebSession
 from freebase.api.session import get_key_namespace, LITERAL_TYPE_IDS
 
+"""
+NOTE
+----
+graph is used freely in this file. Some information:
+ - It refers to an internal representation of a group of types.
+ - It resembles a mqlread result, but it is not a mqlread result
+ - It also has some helper variables like __requires and __related. 
+ - It is produced by _get_graph
+ - It can be converted into valid json (json.dumps(graph, indent=2))
+ 
+Its structure is as follows:
+
+    "type_id" : {
+      "name" : "My Type",
+      "id" : "/base_id/type_id"
+      ...
+      "__requires" : ["/base_id/type_id2"]
+      "__properties" : [
+        {
+          "id" : "/base_id/type_id/my_prop"
+          ...
+        },
+        {
+          ...
+        }
+      ]
+    },
+    "type_id2" : {...}
+    ...
+    
+"""
+
+class DelegationError(Exception):
+    """You can't set the expected_type if the expected_type of the delegated (master) is a primitive"""
+
+class CVTError(Exception):
+    """You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true."""
+
 def key_exists(s, k):
     q = {
         "id" : k,
@@ -74,7 +112,7 @@ def get_property_info(s, prop_id):
             elif res[i].has_key("value"):
                 info[i] = res[i]["value"]
             else:
-                raise Exception("There is a problem with getting the property value.")
+                raise ValueError("There is a problem with getting the property value.")
         else: info[i] = None
 
     # delete the properties that are going to be asked for in create_property
@@ -89,7 +127,7 @@ def get_property_info(s, prop_id):
     # delete other useless things...
     del res["id"]
 
-    for i in [k for k, v in res.items() if v is None]:
+    for i in [k for k, v in res.iteritems() if v is None]:
         del res[i]
 
     info["__raw"] = res
@@ -97,9 +135,17 @@ def get_property_info(s, prop_id):
 
 
 # Create Type
-def create_type(s, name, key, ns, cvt=None, tip=None, included=None, extra=None):
+def create_type(s, name, key, ns, cvt=False, tip=None, included=None, extra=None):
     if key_exists(s, ns + "/" + key ):
         return
+    
+    # assert isinstance(name, basestring) # name could be mqlish
+    assert isinstance(key, basestring)
+    assert isinstance(ns, basestring)
+    
+    assert tip is None or isinstance(tip, basestring)    
+    assert included is None or isinstance(included, (basestring, list, tuple))
+    assert extra is None or isinstance(extra, dict)
 
     q = {
         "create" : "unconditional",
@@ -143,12 +189,17 @@ def create_type(s, name, key, ns, cvt=None, tip=None, included=None, extra=None)
 # Create Property
 def create_property(s, name, key, schema, expected, unique=False, disambig=False, tip=None, extra=None):
     if key_exists(s, schema + "/" + key):
-        # this isn't (neccesarily) a problem
-        # raise Exception("The key \"%s\" already exists!" % (schema + "/" + key))
         return
 
-    # validate parameters str
-
+    
+    # validate parameters
+    # assert isinstance(name, basestring) # could be mql
+    assert isinstance(key, basestring)
+    assert isinstance(schema, basestring)
+    assert isinstance(expected, basestring)
+    assert tip is None or isinstance(tip, basestring)    
+    assert extra is None or isinstance(extra, dict)
+    
     q = {
         "create" : "unconditional",
         "type" : "/type/property",
@@ -169,10 +220,17 @@ def create_property(s, name, key, schema, expected, unique=False, disambig=False
          q['/freebase/property_hints/disambiguator'] = { "connect" : "update", "value" : True }
     if extra:
          q.update(extra)
-    #print json.dumps(q, indent=2)
     return s.mqlwrite(q, use_permission_of=schema)
 
 def delegate_property(s, p, schema, name=None, key=None, expected=None, tip=None, extra=None):
+    
+    assert isinstance(p, basestring)
+    assert isinstance(schema, basestring)
+    #assert name is None or isinstance(name, basestring)
+    assert key is None or isinstance(key, basestring)
+    assert expected is None or isinstance(expected, basestring)
+    assert tip is None or isinstance(tip, basestring)
+    
     q = {
         "id" : p,
         "type" : "/type/property",
@@ -190,7 +248,7 @@ def delegate_property(s, p, schema, name=None, key=None, expected=None, tip=None
     if r["expected_type"]["id"] in LITERAL_TYPE_IDS:
         if expected:
             if expected != r["expected_type"]["id"]:
-                raise Exception("You can't set the expected_type if the expected_type of the delegated (master) is a primitive")
+                raise DelegationError("You can't set the expected_type if the expected_type of the delegated (master) is a primitive")
         expected = r["expected_type"]["id"]
     # If the expected_type of the delegator(master) is not a primitive, the delegated's
     # expected_type can be different
@@ -233,6 +291,13 @@ def reciprocate_property(s, name, key, master, unique=False, disambig=False, tip
     So, given a master, the reciprocal's schema is the master's expected type and the reciprocal's
     expected type is the master's schema. """
 
+    # assert isinstance(name, basestring) # name could be mqlish
+    assert isinstance(key, basestring)
+    assert isinstance(master, basestring)
+    
+    assert tip is None or isinstance(tip, basestring)
+    assert extra is None or isinstance(extra, dict)
+    
 
     q = {
         "id" : master,
@@ -249,174 +314,126 @@ def reciprocate_property(s, name, key, master, unique=False, disambig=False, tip
     return create_property(s, name, key, ect, schema, unique, disambig, tip,
         extra = master)
 
-# upload / restore types
+# dump / restore types
 def dump_base(s, base_id):
-    types = map(lambda x: x["id"], s.mqlread({"id" : base_id, "/type/domain/types":[{"id" : None}]})["/type/domain/types"])
+    types = [type_object["id"] for type_object in s.mqlread({"id" : base_id, "/type/domain/types":[{"id" : None}]})["/type/domain/types"]]
     graph = _get_graph(s, types, True)
-    graph["__follow_types"] = True
-    
+        
     return graph
 
 def dump_type(s, type_id, follow_types=True):
     types = [type_id]
     graph = _get_graph(s, types, follow_types)
-    graph["__follow_types"] = follow_types
     
     return graph
     
 
-def restore(s, graph, new_location, ignore_types=None, debug=False):
+def restore(s, graph, new_location, ignore_types=None):
     follow_types = graph.get("__follow_types", True)
-    if debug: print "Following types:", follow_types
     
     # create type dependencies
-    typegraph = {}
-    for tid, idres in graph.items():
-        if not tid.startswith("__"):
-            typegraph[tid] = idres["__requires"]
-    
-    type_deps = map(lambda (name, x): (len(x), name), typegraph.iteritems())
-    type_deps.sort()
-    if follow_types:
-        types_to_create = create_what(type_deps, typegraph)
-    else:
-        types_to_create = typegraph.keys()
+    type_requires_graph = {}
     
-    # create property dependencies
-    propgraph = {}
-    proptotype = {}
-    for tid, idres in graph.items():
-        if not tid.startswith("__"):
-            for prop in idres["properties"]:
-                propgraph[prop["id"]] = prop["__requires"]
-                proptotype[prop["id"]] = tid
-    prop_deps = map(lambda (name, x): (len(x), name), propgraph.items()) #*
-    prop_deps.sort()
-    if follow_types:
-        props_to_create = create_what(prop_deps, propgraph)
-    else:
-        props_to_create = propgraph.keys()
+    # create prop dependencies
+    prop_requires_graph = {}
+    prop_to_type_map = {}
+        
+    for type_id, type_information in graph.iteritems():
+        if not type_id.startswith("__"): # not a real type, but rather a helper
+            # type dependency generation
+            type_requires_graph[type_id] = type_information["__requires"]
+            
+            # prop dependency generation
+            for prop in type_information["properties"]:
+                prop_requires_graph[prop["id"]] = prop["__requires"]
+                prop_to_type_map[prop["id"]] = type_id
+            
     
-    if debug: print "types", types_to_create
-    if debug: print "-----------------------"
-    if debug: print "props", props_to_create
     
-    base_id, domain_id = s.mqlreadmulti([{"id" : types_to_create[0], "type" : "/type/type", "domain" : {"id" : None}},
-                                         {"id" : new_location, "a:id" : None}])                         
-    base_id = base_id["domain"]["id"]
-    domain_id = domain_id["a:id"]
+    types_to_create = _generate_dependency_creation_order(type_requires_graph)
+    props_to_create = _generate_dependency_creation_order(prop_requires_graph)
+        
+    origin_id, new_location_id = s.mqlreadmulti([{"id" : types_to_create[0], "type" : "/type/type", "domain" : {"id" : None}},
+                                               {"id" : new_location, "a:id" : None}])                         
+    origin_id = origin_id["domain"]["id"]
+    new_location_id = new_location_id["a:id"]
     
     only_include = types_to_create + props_to_create
     
-    for type in types_to_create:
-        if debug: print type
-        key = ""
-        if len(graph[type]["key"]) == 1:
-            key = graph[type]["key"][0]["value"]
-        else:
-            expectedname = graph[type]["id"].split("/")[-1]
-            if base_id:
-                for group in graph[type]["key"]:
-                    if group["namespace"] == base_id:
-                        key = group["value"]
-                        continue
-            if key is None:
-                key = expectedname
+    for type_id in types_to_create:
+        # let's find the type's key
+        key = None
+        for group in graph[type_id]["key"]:
+            if group["namespace"] == origin_id:
+                key = group["value"]
+                break
+        
+        if key is None: # this shouldn't happen
+            key = graph[type_id]["id"].split("/")[-1] # this can be wrong... different key than typeid
+            
+        
         tip = None
-        if graph[type]["/freebase/documented_object/tip"]:
-            tip = graph[type]["/freebase/documented_object/tip"]["value"]
+        if graph[type_id]["/freebase/documented_object/tip"]:
+            tip = graph[type_id]["/freebase/documented_object/tip"]["value"]
         
         ignore = ("name", "domain", "key", "type", "id", "properties", "/freebase/type_hints/enumeration",
                     "/freebase/type_hints/included_types", "/freebase/type_hints/mediator", "/freebase/documented_object/tip")
-        extra = {}
-        for k, v in graph[type].items():
-            if k not in ignore and not k.startswith("__"):
-                if v:
-                    if isinstance(v, basestring):
-                        extra.update({k:v})
-                    elif isinstance(v, bool):
-                        extra.update({k:v})
-                    elif v.has_key("id"):
-                        extra.update({k:v["id"]})
-                    elif v.has_key("value"):
-                        extra.update({k:v["value"]})
-                    else:
-                        raise Exception("There is a problem with getting the property value.")
+        extra = _generate_extra_properties(graph[type_id], ignore)
         
-        create_type(s, graph[type]["name"]["value"], key, domain_id,
-            included=map(lambda x: convert_name(x["id"], base_id, domain_id, only_include), graph[type]["/freebase/type_hints/included_types"]),
-            cvt=graph[type]["/freebase/type_hints/mediator"],
-            tip=tip, extra=extra)
-
-    
-    if debug: print "--------------------------"
+        name = graph[type_id]["name"]["value"]
+        included = [_convert_name_to_new(included_type["id"], origin_id, new_location_id, only_include) for included_type in graph[type_id]["/freebase/type_hints/included_types"]]
+        cvt = graph[type_id]["/freebase/type_hints/mediator"]
+        
+        create_type(s, name, key, new_location_id, included=included, cvt=cvt, tip=tip, extra=extra)
     
-    for prop in props_to_create: #* prop_id
-        info = graph[proptotype[prop]]["properties"]
-        for i in info:
-            if i["id"] == prop: 
+    for prop_id in props_to_create: #* prop_id
+        type_id = prop_to_type_map[prop_id]
+        all_properties_for_type = graph[type_id]["properties"]
+        for prop in all_properties_for_type:
+            if prop["id"] == prop_id: # good, we are dealing with our specific property 
+                
+                new_schema = _convert_name_to_new(type_id, origin_id, new_location_id, only_include)
+                
+                name =  prop["name"]
                 
-                schema = convert_name(proptotype[prop], base_id, domain_id, only_include)
-                if debug: print prop
                 expected = None
+                if prop["expected_type"]:
+                    expected = _convert_name_to_new(prop["expected_type"], origin_id, new_location_id, only_include)
                 
-                if i["expected_type"]:
-                    expected = convert_name(i["expected_type"], base_id, domain_id, only_include)
-                for k in i["key"]:
-                    if k["namespace"] == proptotype[prop]:
-                        key = k["value"]
-                if i["/freebase/documented_object/tip"]:
-                    tip = graph[type]["/freebase/documented_object/tip"]
+                for group in prop["key"]:
+                    if group["namespace"] == type_id:
+                        key = group["value"]
+                        break
+                
+                tip = None
+                if prop["/freebase/documented_object/tip"]:
+                    tip = prop["/freebase/documented_object/tip"]["value"]
                 
-                disambig = i["/freebase/property_hints/disambiguator"]
+                disambig = prop["/freebase/property_hints/disambiguator"]
+                unique = prop["unique"]
                 
                 ignore = ("name", "expected_type", "key", "id", "master_property", "delegated", "unique", "type", "schema",
                             "/freebase/property_hints/disambiguator", "enumeration", "/freebase/property_hints/enumeration", 
                             "/freebase/documented_object/tip")
-                extra = {}
-                for k, v in i.items():
-                    if k not in ignore and not k.startswith("__"):
-                        if v:
-                            if isinstance(v, basestring):
-                                extra.update({k:v})
-                            elif isinstance(v, bool):
-                                extra.update({k:v})
-                            elif v.has_key("id"):
-                                extra.update({k:v["id"]})
-                            elif v.has_key("value"):
-                                extra.update({k:v["value"]})
-                            else:
-                                raise Exception("There is a problem with getting the property value.")
-                            
-                            # since we are creating a property, all these connect insert delicacies are unneccesary    
-                            """if isinstance(v, basestring) and v.startswith("/"): # an id
-                                extra.update({k : {"connect" : "insert", "id" : v}})
-                            elif isinstance(v, bool): # a bool value
-                                extra.update({k : {"connect" : "insert", "value" : v}})
-                            else: # an english value
-                                extra.update({k : {"connect" : "insert", "value" : v, "lang" : "/lang/en"}})"""
                 
+                extra = _generate_extra_properties(prop, ignore)   
                 
-                if i['master_property']:
-                    converted_master_property = convert_name(i["master_property"], base_id, domain_id, only_include)
-                    if converted_master_property == i["master_property"]:
-                        raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
+                if prop['master_property']:
+                    converted_master_property = _convert_name_to_new(prop["master_property"], origin_id, new_location_id, only_include)
+                    if converted_master_property == prop["master_property"]:
+                        raise CVTError("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
                                         "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
-                    reciprocate_property(s, i["name"], key, converted_master_property,
-                        i["unique"], disambig=disambig, tip=tip, extra=extra)
+                    reciprocate_property(s, name, key, converted_master_property,
+                        unique, disambig=disambig, tip=tip, extra=extra)
                 
-                elif i['delegated']:
-                    delegate_property(s, convert_name(i['delegated'], base_id, domain_id, only_include), schema,
+                elif prop['delegated']:
+                    delegate_property(s, _convert_name_to_new(prop['delegated'], origin_id, new_location_id, only_include), new_schema,
                         expected=expected, tip=tip, extra=extra)
                 
                 else:
-                    create_property(s, i["name"], key, schema, expected, i["unique"], 
+                    create_property(s, name, key, new_schema, expected, unique, 
                         disambig=disambig, tip=tip, extra=extra) 
                         
-                
-        
-    
-
 
 def _get_graph(s, initial_types, follow_types):
     """ get the graph of dependencies of all the types involved, starting with a list supplied """
@@ -434,40 +451,73 @@ def _get_graph(s, initial_types, follow_types):
             done.update(graph[new]["__related"])
         if not follow_types:
             # we have to check that there are no cvts attached to us, or else
-            # ugly things happen (we can't include the cvt)
+            # ugly things happen (we can't include the cvt because the cvt won't link to us.)
             for prop in graph[new]["properties"]:
                 if prop["master_property"]:
-                    raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
+                    raise CVTError("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
                                     "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
+    
+    graph["__follow_types"] = follow_types
     return graph
 
 
-def convert_name(old_name, operating_base, new_base, only_include=None):
+def _convert_name_to_new(old_name, operating_base, new_base, only_include=None):
     if old_name in only_include and old_name.startswith(operating_base):
         return new_base + old_name.replace(operating_base, "", 1)
     else:
         return old_name
 
-def create_what(deps, graph):
-    create_list = []
-    while len(deps) > 0:
-        neediness, id = deps.pop(0)
-        if neediness == 0:
-            create_list.append(id)
+def _generate_dependency_creation_order(requires_graph):
+    # This is a naive topographical sort to determine
+    # in what order to create types or properties so
+    # that the other type/properties they rely on
+    # are already created
+    
+    # This function is called with the type dependencies
+    # and then the property dependencies.
+    
+    
+    # we sort the dependency_list because its a good idea
+    # to create the guys with zero dependencies before the
+    # guys with one.. it's just a simple optimization to 
+    # the topographical sort
+    dependency_list = [(len(requires), name) for (name, requires) in requires_graph.iteritems()]
+    dependency_list.sort()
+    
+    creation_order_list = []
+    while len(dependency_list) > 0:
+        number_of_requirements, id = dependency_list.pop(0)
+        if number_of_requirements == 0:
+            creation_order_list.append(id)
             continue
         else:
-            work = True
-            for req in graph[id]:
-                if req not in create_list:
-                    work = False
+            are_the_types_dependencies_already_resolved = True
+            for requirement in requires_graph[id]:
+                if requirement not in creation_order_list:
+                    are_the_types_dependencies_already_resolved = False
                     continue
-            if work:
-                create_list.append(id)
+            if are_the_types_dependencies_already_resolved:
+                creation_order_list.append(id)
             else:
-                deps.append((neediness, id))
-    return create_list        
-
+                dependency_list.append((number_of_requirements, id))
+    return creation_order_list
 
+def _generate_extra_properties(dictionary_of_values, ignore):
+    extra = {}
+    for k, v in dictionary_of_values.iteritems():
+        if k not in ignore and not k.startswith("__"):
+            if v:
+                if isinstance(v, basestring):
+                    extra.update({k:v})
+                elif isinstance(v, bool):
+                    extra.update({k:v})
+                elif v.has_key("id"):
+                    extra.update({k:v["id"]})
+                elif v.has_key("value"):
+                    extra.update({k:v["value"]})
+                else:
+                    raise ValueError("There is a problem with getting the property value.")
+    return extra
 
 def _get_needed(s, type_id):
     q = TYPE_QUERY
@@ -491,8 +541,8 @@ def _get_needed(s, type_id):
     # related can be used to generate subgraphs of types
     # required is used to generate the dependency graph of types
 
-    related = return_relevant(related_types, parents)
-    requires = return_relevant(included_types, parents)
+    related = _return_relevant(related_types, parents)
+    requires = _return_relevant(included_types, parents)
 
     # get property information
     properties = r["properties"]
@@ -503,7 +553,7 @@ def _get_needed(s, type_id):
         if prop["delegated"]:
             dependent_on.add(prop["delegated"])
 
-        prop["__requires"] = return_relevant(dependent_on, parents)
+        prop["__requires"] = _return_relevant(dependent_on, parents)
 
     # return all the information along with our special __* properties
     info = r
@@ -512,7 +562,7 @@ def _get_needed(s, type_id):
     return info
 
 
-def return_relevant(start_list, parents):
+def _return_relevant(start_list, parents):
     final = []
     for item in start_list:
         indomain = False
diff --git a/freebase/schema_cmd.py b/freebase/schema_cmd.py
new file mode 100644
index 0000000..a4644e8
--- /dev/null
+++ b/freebase/schema_cmd.py
@@ -0,0 +1,162 @@
+from optparse import OptionParser
+import getpass
+import sys
+
+from freebase.api import HTTPMetawebSession
+
+from freebase.schema import dump_base, dump_type, restore
+
+try:
+    import jsonlib2 as json
+except ImportError:
+    try:
+        import simplejson as json
+    except ImportError:
+        import json
+
+def fb_save_base():
+    op = OptionParser(usage='%prog [options] baseid')
+    
+    op.disable_interspersed_args()
+    
+    op.add_option('-s', '--service', dest='service_host',
+                    metavar='HOST',
+                    default="freebase.com",
+                    help='Freebase HTTP service address:port')
+    
+    op.add_option('-S', '--sandbox', dest='use_sandbox',
+                    default=False, action='store_true',
+                    help='shortcut for --service=sandbox-freebase.com')
+
+    options, args = op.parse_args()
+    
+    service_host = options.service_host
+    if options.use_sandbox:
+        service_host = "sandbox-freebase.com"
+
+    if len(args) < 1:
+        op.error('Required baseid missing')
+    
+    if len(args) > 1:
+        op.error('Too many arguments')
+    
+    s = HTTPMetawebSession(service_host)
+    
+    print json.dumps(dump_base(s, args[0]), indent=2)
+    
+def fb_save_type():
+    op = OptionParser(usage='%prog [options] typeid ')
+    
+    op.disable_interspersed_args()
+
+    op.add_option('-s', '--service', dest='service_host',
+                    metavar='HOST',
+                    default="freebase.com",
+                    help='Freebase HTTP service address:port')
+    
+    op.add_option('-S', '--sandbox', dest='use_sandbox',
+                    default=False, action='store_true',
+                    help='shortcut for --service=sandbox-freebase.com')
+    
+    op.add_option('-n', '--no-follow', dest='follow',
+                    default=False, action='store_false',
+                    help="Don't follow types, only copy the one specified.")
+    
+    op.add_option('-f', '--follow', dest="follow",
+                    default=True, action="store_true",
+                    help="Follow the types (you might end up copying multiple types)")
+
+
+    options,args = op.parse_args()
+    
+    service_host = options.service_host
+    if options.use_sandbox:
+        service_host = "sandbox-freebase.com"
+
+    if len(args) < 1:
+        op.error('Required typeid missing')
+    
+    if len(args) > 1:
+        op.error('Too many arguments')
+    
+    s = HTTPMetawebSession(service_host)
+    print json.dumps(dump_type(s, args[0], options.follow), indent=2)
+    
+
+def fb_restore():
+    op = OptionParser(usage='%prog [options] new_location graph_output_from_dump*_command')
+    
+    op.disable_interspersed_args()
+
+    op.add_option('-s', '--service', dest='service_host',
+                    metavar='HOST',
+                    help='Freebase HTTP service address:port')
+    
+    op.add_option('-S', '--sandbox', dest='use_sandbox',
+                    default=False, action='store_true',
+                    help='shortcut for --service=sandbox-freebase.com (default)')
+    
+    op.add_option('-F', '--freebase', dest='use_freebase',
+                    default=False, action='store_true',
+                    help='shortcut for --service=freebase.com (not default)')
+
+    op.add_option('-u', '--username', dest='username',
+                    action='store',
+                    help='username for freebase service')
+
+    op.add_option('-p', '--password', dest='password',
+                    action='store',
+                    help='password for freebase service')
+
+                    
+
+    options,args = op.parse_args()
+
+    if (options.username and not options.password) or (not options.username and options.password):
+        op.error("You must supply both a username and password")
+
+    if options.use_sandbox and options.use_freebase:
+        op.error("You can't use both freebase and sandbox!")
+    
+    if options.service_host and (options.use_sandbox or options.use_freebase):
+        op.error("You can't specify both --service and --freebase or --sandbox")
+    
+    if not options.service_host and not options.use_sandbox and not options.use_freebase:
+        op.error("You have to specify to upload to sandbox or production (freebase)")
+    
+    service_host = options.service_host
+    if options.use_sandbox:
+        service_host = "sandbox-freebase.com"
+    if options.use_freebase:
+        service_host = "freebase.com"
+
+    s = login(service_host, username=options.username, password=options.password)
+    
+    newlocation = args[0]
+    if len(args) == 1:
+        graphfile = "-" #stdin
+    else: graphfile = args[1]
+    if graphfile != "-":
+        fg = open(graphfile, "r")
+        graph = json.load(fg)
+        fg.close()
+    if graphfile == "-": # use stdin
+        graph = json.load(sys.stdin)
+    print "loaded graph", graph
+    restore(s, graph, newlocation, ignore_types=None, debug=False)
+
+def login(api_host, username=None, password=None):
+    
+    s = HTTPMetawebSession(api_host)
+    if not username:
+        print "In order to perform this operation, we need to use a valid freebase username and password"
+        username = raw_input("Please enter your username: ")
+        try:
+            password = getpass.getpass("Please enter your password: ")
+        except getpass.GetPassWarning:
+            password = raw_input("Please enter your password: ")
+
+    s.login(username, password)
+
+    print "Thanks!"
+    return s
\ No newline at end of file
diff --git a/schema-manipulation/get_type.py b/schema-manipulation/get_type.py
deleted file mode 100644
index 80def68..0000000
--- a/schema-manipulation/get_type.py
+++ /dev/null
@@ -1,448 +0,0 @@
-# TODO: copy /freebase/type_hints junk
-# TODO: preserve property order.
-
-from pprint import pprint
-import random
-import json
-
-from freebase.schema import TYPE_QUERY, PROPERTY_QUERY
-from freebase.api  import HTTPMetawebSession, MetawebError
-
-from freebase.schema import create_type, create_property, reciprocate_property, delegate_property
-
-s = HTTPMetawebSession("http://sandbox-freebase.com")
-s.login("nitromaster101@gmail.com", "something")
-
-import time
-
-def dump_base(s, base_id):
-    types = map(lambda x: x["id"], s.mqlread({"id" : base_id, "/type/domain/types":[{"id" : None}]})["/type/domain/types"])
-    graph = _get_graph(types)
-    graph["__follow_types"] = True
-    
-    return graph
-
-def dump_type(s, type_id, follow_types=True):
-    types = [type_id]
-    graph = _get_graph(types, follow_types)
-    graph["__follow_types"] = follow_types
-    result = json.dumps(graph, indent=2)
-    
-    fh = open("junk.json", "w")
-    fh.write(result)
-    fh.close()
-    
-    return graph
-    
-
-def upload_type(s, graph, new_location, ignore_types=None, debug=False):
-    follow_types = graph.get("__follow_types", True)
-    if debug: print "Following types:", follow_types
-    
-    # create type dependencies
-    typegraph = {}
-    for tid, idres in graph.items():
-        if not tid.startswith("__"):
-            typegraph[tid] = idres["__requires"]
-    
-    type_deps = map(lambda (name, x): (len(x), name), typegraph.items())
-    type_deps.sort()
-    if follow_types:
-        types_to_create = create_what(type_deps, typegraph)
-    else:
-        types_to_create = typegraph.keys()
-    
-    # create property dependencies
-    propgraph = {}
-    proptotype = {}
-    for tid, idres in graph.items():
-        if not tid.startswith("__"):
-            for prop in idres["properties"]:
-                propgraph[prop["id"]] = prop["__requires"]
-                proptotype[prop["id"]] = tid
-    prop_deps = map(lambda (name, x): (len(x), name), propgraph.items())
-    prop_deps.sort()
-    if follow_types:
-        props_to_create = create_what(prop_deps, propgraph)
-    else:
-        props_to_create = propgraph.keys()
-    
-    if debug: print "types", types_to_create
-    if debug: print "-----------------------"
-    if debug: print "props", props_to_create
-    
-    base_id, domain_id = s.mqlreadmulti([{"id" : types_to_create[0], "type" : "/type/type", "domain" : {"id" : None}},
-                                         {"id" : new_location, "a:id" : None}])                         
-    base_id = base_id["domain"]["id"]
-    domain_id = domain_id["a:id"]
-    
-    only_include = types_to_create + props_to_create
-    
-    for type in types_to_create:
-        if debug: print type
-        key = ""
-        if len(graph[type]["key"]) == 1:
-            key = graph[type]["key"][0]["value"]
-        else:
-            expectedname = graph[type]["id"].split("/")[-1]
-            if base_id:
-                for group in graph[type]["key"]:
-                    if group["namespace"] == base_id:
-                        key = group["value"]
-                        continue
-            if key is None:
-                key = expectedname
-        tip = None
-        if graph[type]["/freebase/documented_object/tip"]:
-            tip = graph[type]["/freebase/documented_object/tip"]["value"]
-        
-        ignore = ("name", "domain", "key", "type", "id", "properties", "/freebase/type_hints/enumeration",
-                    "/freebase/type_hints/included_types", "/freebase/type_hints/mediator", "/freebase/documented_object/tip")
-        extra = {}
-        for k, v in graph[type].items():
-            if k not in ignore and not k.startswith("__"):
-                if v:
-                    if isinstance(v, basestring):
-                        extra.update({k:v})
-                    elif isinstance(v, bool):
-                        extra.update({k:v})
-                    elif v.has_key("id"):
-                        extra.update({k:v["id"]})
-                    elif v.has_key("value"):
-                        extra.update({k:v["value"]})
-                    else:
-                        raise Exception("There is a problem with getting the property value.")
-        
-        create_type(s, graph[type]["name"]["value"], key, domain_id,
-            included=map(lambda x: convert_name(x["id"], base_id, domain_id, only_include), graph[type]["/freebase/type_hints/included_types"]),
-            cvt=graph[type]["/freebase/type_hints/mediator"],
-            tip=tip, extra=extra)
-
-    
-    if debug: print "--------------------------"
-    
-    for prop in props_to_create:
-        info = graph[proptotype[prop]]["properties"]
-        for i in info:
-            if i["id"] == prop: 
-                
-                schema = convert_name(proptotype[prop], base_id, domain_id, only_include)
-                if debug: print prop
-                expected = None
-                
-                if i["expected_type"]:
-                    expected = convert_name(i["expected_type"], base_id, domain_id, only_include)
-                for k in i["key"]:
-                    if k.namespace == proptotype[prop]:
-                        key = k.value
-                if i["/freebase/documented_object/tip"]:
-                    tip = graph[type]["/freebase/documented_object/tip"]
-                
-                disambig = i["/freebase/property_hints/disambiguator"]
-                
-                ignore = ("name", "expected_type", "key", "id", "master_property", "delegated", "unique", "type", "schema",
-                            "/freebase/property_hints/disambiguator", "enumeration", "/freebase/property_hints/enumeration", 
-                            "/freebase/documented_object/tip")
-                extra = {}
-                for k, v in i.items():
-                    if k not in ignore and not k.startswith("__"):
-                        if v:
-                            if isinstance(v, basestring):
-                                extra.update({k:v})
-                            elif isinstance(v, bool):
-                                extra.update({k:v})
-                            elif v.has_key("id"):
-                                extra.update({k:v["id"]})
-                            elif v.has_key("value"):
-                                extra.update({k:v["value"]})
-                            else:
-                                raise Exception("There is a problem with getting the property value.")
-                            
-                            # since we are creating a property, all these connect insert delicacies are unneccesary    
-                            """if isinstance(v, basestring) and v.startswith("/"): # an id
-                                extra.update({k : {"connect" : "insert", "id" : v}})
-                            elif isinstance(v, bool): # a bool value
-                                extra.update({k : {"connect" : "insert", "value" : v}})
-                            else: # an english value
-                                extra.update({k : {"connect" : "insert", "value" : v, "lang" : "/lang/en"}})"""
-                
-                
-                if i['master_property']:
-                    converted_master_property = convert_name(i["master_property"], base_id, domain_id, only_include)
-                    if converted_master_property == i["master_property"]:
-                        raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
-                                        "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
-                    reciprocate_property(s, i["name"], key, converted_master_property,
-                        i["unique"], disambig=disambig, tip=tip, extra=extra)
-                
-                elif i['delegated']:
-                    delegate_property(s, convert_name(i['delegated'], base_id, domain_id, only_include), schema,
-                        expected=expected, tip=tip, extra=extra)
-                
-                else:
-                    create_property(s, i["name"], key, schema, expected, i["unique"], 
-                        disambig=disambig, tip=tip, extra=extra) 
-                        
-                
-        
-    
-
-
-def _get_graph(initial_types, follow_types):
-    """ get the graph of dependencies of all the types involved, starting with a list supplied """
-    
-    assert isinstance(initial_types, (list, tuple))
-    
-    graph = {}
-    to_update = set(initial_types)
-    done = set()
-    while len(to_update) > 0:
-        new = to_update.pop()
-        graph[new] = _get_needed(s, new)
-        if follow_types:
-            [to_update.add(b) for b in graph[new]["__related"] if b not in done]
-            done.update(graph[new]["__related"])
-        if not follow_types:
-            # we have to check that there are no cvts attached to us, or else
-            # ugly things happen (we can't include the cvt)
-            for prop in graph[new]["properties"]:
-                if prop["master_property"]:
-                    raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
-                                    "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
-    return graph
-
-"""def create_type_dependencies(s, base_id=None, type_id=None, follow_types=True):
-    
-    if base_id:
-        q = {"id" : base_id, "/type/domain/types" : [{"id" : None}] }
-        results = s.mqlread(q)
-        types = map(lambda x: x["id"], results["/type/domain/types"])
-        #pprint(types)
-    
-    if type_id:
-        types = [type_id]
-        base_id = "/".join(type_id.split("/")[:-1])
-        print base_id
-    
-    if not base_id and not type_id:
-        raise Exception("You need to supply either a base_id or a type_id")
-    
-    graph = get_graph(types, follow_types)
-
-    # find distinct subgraphs (looking at needs)
-    subgraphs = []
-    unknown = set(graph.keys())
-    if focus:
-        unknown = set([focus])
-    while len(unknown) > 0:
-        visited = set()
-        to_visit = set([list(unknown)[0]])
-        while len(to_visit) > 0:
-            new = to_visit.pop()
-            visited.add(new)
-            try:
-                unknown.remove(new)
-            except KeyError:
-                pass
-            [to_visit.add(b) for b in graph[new]["related"] if b not in visited]
-        subgraphs.append(list(visited))
-
-    
-    print "SUBGRPAHS"
-    pprint(subgraphs)
-    print "GRAPH"
-    pprint(graph)
-    return
-    
-    # create type dependencies
-    typegraph = {}
-    for tid, idres in graph.items():
-        typegraph[tid] = idres["needs"]
-    
-    least_needy_type = map(lambda (name, x): (len(x), name), typegraph.items())
-    least_needy_type.sort()
-    
-    types_to_create = create_what(least_needy_type, typegraph)
-    
-    # create property dependencies
-    propgraph = {}
-    proptotype = {}
-    for tid, idres in graph.items():
-        for prop in idres["properties"]:
-            propgraph[prop["id"]] = prop["needs"]
-            proptotype[prop["id"]] = tid
-    least_needy = map(lambda (name, x): (len(x), name), propgraph.items())
-    least_needy.sort()
-    
-    props_to_create = create_what(least_needy, propgraph)
-    
-    # CREATING STUFF
-    
-    domainname = "awesome" + str(int(random.random() * 1e10))
-    print "\ndomainid", domainname
-    print "--------------------------"
-    dn = s.create_private_domain(domainname, domainname + "!")
-    domain_id = dn.domain_id
-    
-    better_id = s.mqlread({"id" : domain_id, "a:id" : None})
-    domain_id = better_id["a:id"]
-    
-    for type in types_to_create:
-        print type
-        key = ""
-        if len(graph[type]["key"]) == 1:
-            key = graph[type]["key"][0]["value"]
-        else:
-            expectedname = graph[type]["id"].split("/")[-1]
-            if base_id:
-                for group in graph[type]["key"]:
-                    if group["namespace"] == base_id:
-                        key = group["value"]
-                        continue
-            if key is None:
-                key = expectedname
-        tip = None
-        if graph[type]["/freebase/documented_object/tip"]:
-            tip = graph[type]["/freebase/documented_object/tip"]["value"]
-        create_type(s, graph[type]["name"]["value"], key, domain_id,
-            included=map(lambda x: convert_name(x["id"], base_id, domain_id), graph[type]["/freebase/type_hints/included_types"]),
-            cvt=graph[type]["/freebase/type_hints/mediator"],
-            enum=graph[type]["/freebase/type_hints/enumeration"],
-            tip=tip)
-    
-    print "--------------------------"
-    
-    for prop in props_to_create:
-        info = graph[proptotype[prop]]["properties"]
-        for i in info:
-            if i["id"] == prop: 
-                schema = convert_name(proptotype[prop], base_id, domain_id)
-                print prop
-                expected = None
-                if i["expected_type"]:
-                    expected = convert_name(i["expected_type"]["id"], base_id, domain_id)
-                for k in i["key"]:
-                    if k.namespace == proptotype[prop]:
-                        key = k.value
-                if i["/freebase/documented_object/tip"]:
-                    tip = graph[type]["/freebase/documented_object/tip"]
-                    
-                if i['master_property']:
-                    reciprocate_property(s, i["name"], key, convert_name(i["master_property"]["id"], base_id, domain_id),
-                        i["unique"], disambig=i["/freebase/property_hints/disambiguator"], tip=tip)
-                elif i['delegated']:
-                    delegate_property(s, convert_name(i['delegated']['id'], base_id, domain_id), schema,
-                        expected=expected, tip=tip)
-                else:
-                    create_property(s, i["name"], key, schema, expected, i["unique"], 
-                        disambig=i["/freebase/property_hints/disambiguator"], tip=tip) 
-                        
-                
-        
-    
-    print "\ndomain id was", domainname
-"""
-
-
-def convert_name(old_name, operating_base, new_base, only_include=None):
-    if old_name in only_include and old_name.startswith(operating_base):
-        return new_base + old_name.replace(operating_base, "", 1)
-    else:
-        return old_name
-    
-def create_what(deps, graph):
-    create_list = []
-    while len(deps) > 0:
-        neediness, id = deps.pop(0)
-        if neediness == 0:
-            create_list.append(id)
-            continue
-        else:
-            work = True
-            for req in graph[id]:
-                if req not in create_list:
-                    work = False
-                    continue
-            if work:
-                create_list.append(id)
-            else:
-                deps.append((neediness, id))
-    return create_list        
-       
-
-
-def _get_needed(s, type_id):
-    q = TYPE_QUERY
-    q.update(id=type_id)
-    
-    r = s.mqlread(q)
-    properties = r.properties
-    
-    # let's identify who the parent is in order to only include
-    # other types in that domain. We don't want to go around including
-    # all of commons because someone's a /people/person
-    parents = [r["domain"]["id"]]
-   
-    included_types = map(lambda x: x["id"], r["/freebase/type_hints/included_types"])
-    related_types = set(included_types)
-    for prop in properties:
-        if prop["expected_type"]:
-            related_types.add(prop["expected_type"])
-    
-    # we have two different types of relationships: required and related.
-    # related can be used to generate subgraphs of types
-    # required is used to generate the dependency graph of types
-        
-    related = return_relevant(related_types, parents)
-    requires = return_relevant(included_types, parents)
-    
-    # get property information
-    properties = r["properties"]
-    for prop in properties:
-        dependent_on = set()
-        if prop["master_property"]:
-            dependent_on.add(prop["master_property"])
-        if prop["delegated"]:
-            dependent_on.add(prop["delegated"])
-        
-        prop["__requires"] = return_relevant(dependent_on, parents)
-    
-    # return all the information along with our special __* properties
-    info = r
-    info.update(__related=related, __requires=requires, __properties=properties)
-    
-    return info
-    
-
-def return_relevant(start_list, parents):
-    final = []
-    for item in start_list:
-        indomain = False
-        for parent in parents:
-            if item.startswith(parent):
-                indomain = True
-                continue
-        if indomain:
-            final.append(item)
-    return final
-
-if __name__ == '__main__':
-    
-    # create domain
-    name = str(int(random.random()*1e10))
-    domain_id = s.create_private_domain("coolcat" + name, "Coolcat" + name)["domain_id"]
-    domain_id = s.mqlread({"id" : domain_id, "a:id" : None})["a:id"]
-    print "domain name", domain_id
-
-    # dump information
-    #graph = dump_type(s, "/user/nitromaster101/coolcat3194242218/bridge_player", follow_types=True)
-    graph = dump_type(s, "/people/person", True)
-    
-    # upload it somewhere else
-    upload_type(s, graph, domain_id, debug=True)
-    
-    
-    print "domain name", domain_id
-    #create_type_dependencies(s, type_id="/base/contractbridge/bridge_player")
-    
-    
diff --git a/schema-manipulation/type_creation.py b/schema-manipulation/type_creation.py
deleted file mode 100644
index b940076..0000000
--- a/schema-manipulation/type_creation.py
+++ /dev/null
@@ -1,470 +0,0 @@
-try:
-    import simplejson as json
-except ImportError:
-    import json
-
-import urllib
-from freebase.api import HTTPMetawebSession, MetawebError
-
-from freebase.api import LITERAL_TYPE_IDS
-s = HTTPMetawebSession('sandbox.freebase.com', username, password)
-s.login()
-
-def key_exists(s, k):
-    q = {
-        "id" : k,
-        "guid" : None
-    }
-    return not None == s.mqlread(q)
-
-
-# Object helpers
-def create_object(s, name="", path=None, key=None, namespace=None, included_types=[], create="unless_exists", extra=None):
-    if type(included_types) is str:
-        included_types = [included_types]
-    
-    if path and (key or namespace):
-        raise Exception("You can't specify both the path and a key and namespace.")
-    
-    if path:
-        key, namespace = get_key_namespace(path)
-    
-    its = set(included_types)
-    if included_types:
-        q = [{
-            "id|=" : included_types,
-            "/freebase/type_hints/included_types" : [{"id" : None}]
-        }]
-        for res in s.mqlread(q):
-            its.update(map(lambda x: x["id"], res["/freebase/type_hints/included_types"]))
-    
-    wq = {
-        "id" : None,
-        "name" : name,
-        "key" : {
-            "namespace" : namespace,
-            "value" : key,
-        },
-        "create" : create
-    }
-    
-    if included_types:
-        wq.update(type = [{ "id" : it, "connect" : "insert" } for it in its])
-    
-    if extra: 
-        wq.update(extra)
-    
-    return s.mqlwrite(wq)
-
-
-def connect_object(s, id, newpath, extra=None):
-    
-    key, namespace = get_key_namespace(newpath)
-    
-    wq = {
-        "id" : id,
-        "key" : {
-            "namespace" : namespace,
-            "value" : key,
-            "connect" : "insert"
-        }
-    }
-    
-    if extra: wq.update(extra)
-    
-    return s.mqlwrite(wq)
-
-
-def unconnect_object(s, id, extra=None):
-    
-    key, namespace = get_key_namespace(id)
-    
-    wq = {
-        "id" : id,
-        "key" : {
-            "namespace" : namespace,
-            "value" : key,
-            "connect" : "delete"
-        }
-    }
-    if extra: wq.update(extra)
-    return s.mqlwrite(wq)
-
-def move_object(s, oldpath, newpath):
-    a = connect_object(s, oldpath, newpath)
-    b = unconnect_object(s, oldpath)
-    return a, b
-
-def type_object(s, id, type_id):
-    q = {
-        "id" : type_id,
-        "/freebase/type_hints/included_types" : [{"id" : None}]
-    }
-    included_types = map(lambda x: x["id"], s.mqlread(q)["/freebase/type_hints/included_types"])
-    
-    wq = {
-        "id" : id,
-        "type" : [{
-            "id" : it,
-            "connect" : "insert"
-        } for it in included_types + [type_id]]
-    }
-    return s.mqlwrite(wq)
-
-def get_key_namespace(path):
-    split = path.split("/")
-    return split[-1], "/".join(split[:-1]) or "/"
-
-
-# type moving
-
-
-# property moving
-
-def connect_property(s, id, newid, **extra):
-    split = newid.split("/")
-    newschema = "/".join(split[:-1])
-    newname = split[-1]
-    
-    info = get_property_info(s, id)
-    info["__raw"].update(extra)
-    create_property(s, info["name"], newname, newschema, info["expected_type"], info["unique"], info["/freebase/property_hints/disambiguator"],
-        info["/freebase/documented_object/tip"], info["__raw"])  
-
-def move_property(s, id, newid, **extra):
-    connect_property(s, id, newid, **extra)
-    disconnect_schema = {"type" : "/type/property", "schema" : {"connect" : "delete", "id" : "/".join(id.split("/")[:-1]) }}    
-    unconnect_object(s, id, extra = disconnect_schema)
-
-PROPERTY_QUERY = {
-        "optional" : True,
-        "type" : "/type/property",
-        "delegated" : {},
-        "enumeration" : {}, 
-        "expected_type" : {}, 
-        "id" : {},
-        "key" : [{
-            "namespace" : None, 
-            "value" : None
-        }],
-        #"link" : [{}], 
-        "master_property" : {},
-        "name" : {"value" : None, "lang" : "/lang/en", "optional":True}, 
-        "schema" : {"id" : None, "name" : None},
-        "unique" : {}, 
-        "unit" : {},
-        "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional" : True},
-        "/freebase/property_hints/disambiguator" : {},
-        "/freebase/property_hints/display_none" : {},
-        "/freebase/property_hints/display_orientation" : {},
-        "/freebase/property_hints/enumeration" : {},
-        "/freebase/property_hints/dont_display_in_weblinks" : {},
-        "/freebase/property_hints/inverse_description" : {},
-    }
-
-TYPE_QUERY = {
-        "type" : "/type/type",
-        "domain" : {},
-        "key" : [{"namespace" : None, "value" : None}],
-        "name" : {"value" : None, "lang" : "/lang/en", "optional":True},
-        "/freebase/type_hints/included_types" : [{}],
-        "/freebase/type_hints/mediator" : None,
-        "/freebase/type_hints/enumeration" : None,
-        "/freebase/type_hints/minor" : None,
-        "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional":True},
-        "properties" : {"optional" : True}
-        }
-TYPE_QUERY.update(properties=[PROPERTY_QUERY])
-
-
-def get_property_info(s, prop_id):
-    q = PROPERTY_QUERY
-    q.update(id=prop_id)
-    res = s.mqlread(q)
-    info = {}
-
-    info["name"] = res["name"]["value"]
-    if res["schema"]:
-        info["schema"] = res["schema"]["id"]
-    else: info["schema"] = None
-    
-    if res["key"]:
-        info["keys"] = map(lambda x: (x["value"], x["namespace"]), res["key"])
-    else: info["key"] = None
-    
-    if res["/freebase/documented_object/tip"]:
-        info["/freebase/documented_object/tip"] = res["/freebase/documented_object/tip"]["value"]
-    else: info["/freebase/documented_object/tip"] = None
-
-    for i in ["delegated", "enumeration", "expected_type", "id", "master_property", "unique", "unit",
-     "/freebase/property_hints/disambiguator", "/freebase/property_hints/display_none",
-     "/freebase/property_hints/display_orientation","/freebase/property_hints/enumeration",
-      "/freebase/property_hints/dont_display_in_weblinks", "/freebase/property_hints/inverse_description"]:
-
-        if res[i]:
-            if isinstance(res[i], basestring):
-                info[i] = res[i]
-            elif res[i].has_key("id"):
-                info[i] = res[i]["id"]
-            elif res[i].has_key("value"):
-                info[i] = res[i]["value"]
-            else:
-                raise Exception("There is a problem with getting the property value.")
-        else: info[i] = None
-
-    # delete the properties that are going to be asked for in create_property
-    del res["name"]
-    del res["schema"]
-    del res["key"]
-    del res["expected_type"]
-    del res["unique"]
-    del res["/freebase/property_hints/disambiguator"]
-    del res["/freebase/documented_object/tip"]
-
-    # delete other useless things...
-    del res["id"]
-
-    for i in [k for k, v in res.items() if v is None]:
-        del res[i]
-
-    info["__raw"] = res
-    return info
-
-
-# Create Type
-def create_type(s, name, key, ns, cvt=None, tip=None, enum=None, included=[]):
-    if key_exists( s, ns + "/" + key ):
-        return
-    
-    q = {
-        "create" : "unconditional",
-        "type" : "/type/type",
-        "/type/type/domain" : { "connect" : "insert", "id" : ns },
-        "name" : {"connect" : "insert", "value" : name, "lang" : "/lang/en" },
-        "key" : {
-            "connect" : "insert",
-            "value" : key,
-            "namespace" : ns
-        }
-    }
-   
-    if included:
-        if isinstance(included, basestring):
-            included = [included]
-        q = {
-            "id|=" : included,
-            "/freebase/type_hints/included_types" : [{"id" : None}]
-        }
-        r = s.mqlread(q)
-        included_types = set()
-        for i in r:
-            included_types.append(map(lambda x: x["id"], i["/freebase/type_hints/included_types"]))
-        
-        q['/freebase/type_hints/included_types'] = \
-           [ {"connect":"insert",
-                "id":included_id}
-                for included_id in included_types]
-   
-    if enum:
-        pass # TODO. What is enumerator?
-   
-    if cvt:
-        q['/freebase/type_hints/mediator'] = { "connect" : "update", "value" : True }
-    if tip:
-        q['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" }
-    return s.mqlwrite(q, use_permission_of=ns)
-
-
-# Create Property
-def create_property(s, name, key, schema, expected, unique, disambig=False, tip=None, extra=None):
-    if key_exists(s, schema + "/" + key):
-        raise Exception("The key \"%s\" already exists!", schema + "/" + key)
-        return
-    q = {
-        "create" : "unconditional",
-        "type" : "/type/property",
-        "name" : name,
-        "key" : {
-            "connect" : "insert",
-            "value" : key,
-            "namespace" : { "id" : schema },
-        },
-        "schema" : { "connect" : "insert", "id" : schema },
-        "expected_type" : { "connect" : "insert", "id" : expected }
-    }
-    if unique:
-         q['unique'] = { "connect" : "update", "value" : unique }
-    if tip:
-         q['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" }
-    if disambig:
-         q['/freebase/property_hints/disambiguator'] = { "connect" : "update", "value" : True }
-    if extra:
-         q.update(extra)
-    #print json.dumps(q, indent=2)
-    return s.mqlwrite(q, use_permission_of=schema)
-
-def delegate_property(s, p, schema, name=None, key=None, expected=None, tip=None):
-    q = {
-        "id" : p,
-        "type" : "/type/property",
-        "name" : None,
-        "unique" : None,
-        "expected_type" : {"id" : None},
-        "key" : None,
-        "/freebase/documented_object/tip" : None,
-        "/freebase/property_hints/disambiguator" : None
-    }
-    r = s.mqlread(q)
-   
-    # If the expected_type of the delegator(master) is a primitive, the delegated's
-    # expected_type must be the same
-    if r["expected_type"]["id"] in LITERAL_TYPE_IDS:
-        if expected:
-            print "You can't set the expected_type if the expected_type of the delegated(master) is a primitive"
-        expected = r["expected_type"]["id"]
-    # If the expected_type of the delegator(master) is not a primitive, the delegated's
-    # expected_type can be different
-    elif expected is None:
-        expected = r["expected_type"]["id"]
-      
-   
-    if not tip and r["/freebase/documented_object/tip"]:
-        tip = r["/freebase/documented_object/tip"]
-   
-    if name is None:
-        name = r["name"]
-    if key is None:
-        key = r["key"]
-   
-    return create_property(s, name, key, schema, expected, r['unique'],
-        r["/freebase/property_hints/disambiguator"],
-        tip,
-        { "/type/property/delegated" : p})
-
-def reciprocate_property(s, name, key, master, unique, disambig=False, tip=None):
-    """ We're creating a reciprocate property of the master property. Let's illustrate
-    the idea behind the function with an example.
-    
-    Say we examine the /visual_art/art_period_movement/associated_artworks property.
-    An example of an art_period_movement is the Renaissance, and once associated_artworks
-    could be the /en/mona_lisa. In this example, /visual_art/art_period_movement/associated_artworks
-    will be the master property, and /visual_art/artwork/period_or_movement will be the reciprocal.
-    
-    In order to determine the characterists of the reciprocal property, we must examine the master.
-    associated_artworks property's schema is /visual_art/art_period_movement and its expected
-    type is /visual_art/artwork. Notice the similarity to /visual_art/artwork/period_or_movement.
-    period_or_movement's schema is /visual_art/artwork -- art_period_movement's expected type.
-    period_or_movement's expected type is /visual_art/art_period_movement -- art_period_movement's
-    schema!
-    
-    So, given a master, the reciprocal's schema is the master's expected type and the reciprocal's
-    expected type is the master's schema. """
-   
-    
-    q = {
-        "id" : master,
-        "/type/property/expected_type" : None,
-        "/type/property/schema" : None }
-    r = s.mqlread(q)
-    ect = r["/type/property/expected_type"]
-    schema = r["/type/property/schema"]
-    return create_property(s, name, key, ect, schema, unique, disambig, tip,
-        extra = { "master_property" : master })
-
-
-# ?
-def make_subclass(s, sub, super):
-    q = {
-        "id" : super,
-        "type" : "/type/type",
-        "properties" : [{"id" : None}]
-    }
-    r = s.mqlread( q )
-    for p in map(lambda x: x['id'], r['properties'] ):
-        delegate_property(s, p, sub)
-    w = {
-        "id" : sub,
-        "/freebase/type_hints/included_types" : { "connect" : "insert", "id" : super }
-    }
-    s.mqlwrite(w)
-
-def gogo(base="/base/bingbase"):
-   
-    
-   
-    create_type(s, "Bridge Player", "bridge_player", base, tip="A bridge player", included=["/people/person", "/common/topic", base+"/topic"])
-    create_type(s, "Bridge Tournament", "bridge_tournament", base, tip="A bridge tournament", included=["/event/event", "/common/topic", base+"/topic"])
-    create_type(s, "Bridge Tournament Standings", "bridge_tournament_standings", base, cvt=True, tip="Bridge Tournamen Results")
-   
-    player = base + "/bridge_player"
-    tourney = base + "/bridge_tournament"
-    standing = base + "/bridge_tournament_standings"
-   
-    # tournament standings
-    # ideas: create a cvt could be a function, disambig always True, same schema, iono, whatever
-    create_property(s, "Year", "year", standing, "/type/datetime", True, disambig=True, tip="Year")
-    create_property(s, "First Place", "first_place", standing, player, False, disambig=True)
-    create_property(s, "Second Place", "second_place", standing, player, False, disambig=True)
-    create_property(s, "Tournament", "tournament", standing, tourney, True, disambig=True)
-   
-   
-    # tournament
-    reciprocate_property(s, "Standing", "standing", standing + "/tournament", False, disambig=True)
-    create_property(s, "Location", "location", tourney, "/location/citytown", False, disambig=True)
-   
-    # bridge player
-    # standings must be reverses!
-    reciprocate_property(s, "First Place Finish", "first_place_finish", standing + "/first_place", False, False)
-    reciprocate_property(s, "Second Place Finish", "second_place_finish", standing + "/second_place", False, False)
-   
-    #delegator property test
-    delegate_property(mss, "/people/person/date_of_birth", player, "Date of Birth", "db")
-   
-    # for cvt: expected_type: cvt, reciprocated property is the property on the cvt
-
-
-# another test... not sure it works
-def create_chapter(s):
-    create_type( s, "Chapter", "chapter", "/base/testy", True, "A chapter in one of the books" )
-    print "created book!"
-    create_property( s, "Book", "book", "/base/testy/chapter",
-        "/base/testy/book", True, True, "The book of which this chapter is a part" )
-    print "created property!"
-    reciprocate_property( s, "/base/testy/chapter/book",
-        "Chapters", "chapters", "/base/testy/book", False, False, "Chapters in this book" )
-
-def create_edition(s):
-    create_type( s, "Edition", "edition", "/base/testy", True, "An edition of the Aubrey-Maturin series" )
-    create_property( s, "Examples", "examples", "/base/testy/edition",
-        "/book/book_edition", False, False, "Book editions in this series" )
-    w = {
-        "id" : "/base/testy/edition",
-        "/freebase/type_hints/enumeration" : { "value" : True, "connect" : "update" }
-    }
-    s.mqlwrite( w )
-    w = {
-        "create" : "unless_exists",
-        "type" :"/base/testy/edition",
-        "name" : "W. W. Norton Paperback"
-    }
-    s.mqlwrite( w )
-
-def create_page_ref(s):
-    create_type( s, "Page Reference", "page_ref", "/base/testy", True,
-        "A reference to a page in a particular edition" )
-    t = "/base/testy/page_ref"
-    create_property( s, "Page Number", "page", t, "/type/int", True, True )
-    create_property( s, "Edition", "edition", t, "/base/testy/edition", True, True )
-
-def create_mention_properties(s, t):
-    create_property( s, "Circumstances", "circumstances", t, "/type/text", True, True,
-        "Describe the circumstances of the mention" )
-    create_property( s, "Quote", "quote", t, "/type/text", True, True,
-        "A short quote from the mentioned location" )
-    create_property( s, "Chapter", "chapter", t, "/base/testy/chapter", True, True,
-        "The chapter in which the mention takes place" )
-    create_property( s, "Page", "page", t, "/base/testy/page_ref", False, True,
-        "Page number and Edition" )
-    create_property( s, "What", "what", t, "/base/testy/topic", True, True,
-        "What is being mentioned" )
-
diff --git a/test/runtests.py b/test/runtests.py
index c90afb7..db2303b 100644
--- a/test/runtests.py
+++ b/test/runtests.py
@@ -16,18 +16,23 @@ def main():
         created = True
         USERNAME, PASSWORD = getlogindetails.main(create_password_file=True)
     USERNAME, PASSWORD = getlogindetails.main()        
+ 
     
     # run tests
     import test_freebase
     import test_schema_manipulation
+    import test_hardcore_schema_manipulation
 
     s1 = unittest.TestLoader().loadTestsFromTestCase(test_freebase.TestFreebase)
     s2 = unittest.TestLoader().loadTestsFromTestCase(test_schema_manipulation.TestSchemaManipulation)
+    s3 = unittest.TestLoader().loadTestsFromTestCase(test_hardcore_schema_manipulation.TestHardcoreSchemaManipulation)
     
     # This is very strange. If you try to do [s1, s2], thereby running freebase tests first, 
     # two tests in the testschemamanipulation file fail! They fail because of caching issues; if
     # I check on freebase, the changes are actually there. I have racked my mind for explanations.
-    anotherrun = unittest.TestSuite([s1, s2])
+    
+    # this is such a hack, and I'm sorry. The tests run 100% correct individually.
+    anotherrun = unittest.TestSuite([s1, s3, s2])
     
     #run = unittest.TestSuite(suites)
     
@@ -35,6 +40,7 @@ def main():
     if created: os.remove(passwordfile)  
     
     return anotherrun
+    
 
 
 if __name__ == '__main__':
diff --git a/schema-manipulation/test_hardcore_schema_manipulation.py b/test/test_hardcore_schema_manipulation.py
index 3ef04b1..aaecf28 100644
--- a/schema-manipulation/test_hardcore_schema_manipulation.py
+++ b/test/test_hardcore_schema_manipulation.py
@@ -4,9 +4,10 @@ import freebase
 import random
 import time
 
-from freebase.api import HTTPMetawebSession, MetawebError
+import getlogindetails
 
-from get_type import dump_type, dump_base, upload_type
+from freebase.api import HTTPMetawebSession, MetawebError
+from freebase.schema import dump_type, dump_base, restore
 
 USERNAME = 'username'
 PASSWORD = 'password'
@@ -14,6 +15,11 @@ API_HOST = 'sandbox.freebase.com'
 
 s = freebase.api.HTTPMetawebSession(API_HOST)
 
+if USERNAME == "username" and PASSWORD == "password":
+    USERNAME, PASSWORD = getlogindetails.main()
+
+s.login(USERNAME, PASSWORD)
+
 # Sorry, this is just so annoying to type.
 f = lambda x: x["id"]
 
@@ -21,12 +27,12 @@ class TestHardcoreSchemaManipulation(unittest.TestCase):
  
     def test_copy_an_entire_domain(self):
         domain_id = _create_domain()
-        ex_domain_id = "/film" # example domain id
-        ex_domain_type = "actor"
+        ex_domain_id = "/base/contractbridge" # example domain id
+        ex_domain_type = "bridge_player"
         ex_domain_type_id = ex_domain_id + "/" + ex_domain_type
         
         graph = dump_base(s, ex_domain_id)
-        upload_type(s, graph, domain_id)
+        restore(s, graph, domain_id)
         
         newperson, realperson = s.mqlreadmulti([{"id" : domain_id + "/" + ex_domain_type, "/type/type/properties" : {"return" : "count" }}, 
                                                 {"id" : ex_domain_type_id, "/type/type/properties" : {"return" : "count" }}])
@@ -41,28 +47,29 @@ class TestHardcoreSchemaManipulation(unittest.TestCase):
         
         realtypes = l(realtypes)
         newtypes  = l(newtypes)
-        print realtypes
-        print newtypes
+        
         self.assertEqual(len(realtypes), len(newtypes))
         for i in range(len(realtypes)):
-            self.assertEqual(realtypes[i].lsplit("/", 1), newtypes[i].lsplit("/", 1))
+            self.assertEqual(realtypes[i].rsplit("/", 1)[-1], newtypes[i].rsplit("/", 1)[-1])
         
         # - check the properties are the same
         
         def get_properties(types):
             properties = set()
-            for i in s.mqlreadmulti([[{"id" : id, "/type/type/properties" : {"id" : None}}] for id in types]):
-                properties.update(map(f, i["/type/type/properties"]))
+            for i in s.mqlreadmulti([{"id" : id, "/type/type/properties" : [{"id" : None}]} for id in types]):
+                properties.update(map(lambda x: x["id"], i["/type/type/properties"]))
             return properties
         
         realproperties = sorted(list(get_properties(realtypes)))
         newproperties  = sorted(list(get_properties(newtypes)))
         
-        self.assertEqual(realproperties, newproperties)
-        for i in range(len(realproperties)):
-            self.assertEqual(realtypes[i].lsplit("/", 1), newtypes[i].lsplit("/", 1))
-            
-        # - check the properties and type's attributes are the same 
+        def ignore_base(id): return id.rsplit("/", 1)[-1]
+        
+        self.assertEqual(len(realproperties), len(newproperties))
+        self.assertEqual([ignore_base(prop_id) for prop_id in realproperties], [ignore_base(prop_id) for prop_id in newproperties])
+        
+        # - check the properties and type's attributes are the same
+        
         
         
     
@@ -71,14 +78,15 @@ class TestHardcoreSchemaManipulation(unittest.TestCase):
         # if follow_types is True, everything is kosher.
         domain_id = _create_domain()
         graph = dump_type(s, "/film/actor", follow_types=True)
-        upload_type(s, graph, domain_id)
+        restore(s, graph, domain_id)
         
         newactor, realactor = s.mqlreadmulti([{"id" : domain_id + "/actor", "/type/type/properties" : {"return" : "count" }}, 
-                                    {"id" : "/film/actor", "/type/type/properties" : {"return" : "count" }}])
+                                              {"id" : "/film/actor", "/type/type/properties" : {"return" : "count" }}])
         self.assertEqual(newactor["/type/type/properties"], realactor["/type/type/properties"])
         
         # if follow_types is False, if we try to upload a cvt, it should whine
-        self.assertRaises(Exception, lambda: dump_type(s, "/film/actor", follow_types=False))
+        from freebase.schema import CVTError
+        self.assertRaises(CVTError, lambda: dump_type(s, "/film/actor", follow_types=False))
         
         
 
diff --git a/test/test_schema_manipulation.py b/test/test_schema_manipulation.py
index 53ce542..f8453e7 100644
--- a/test/test_schema_manipulation.py
+++ b/test/test_schema_manipulation.py
@@ -31,30 +31,20 @@ f = lambda x: x["id"]
 class TestSchemaManipulation(unittest.TestCase):
     
     def test_make_and_type_object(self):
-        print domain_id
         a = s.create_object("A", path=domain_id + "/a")
         self.assertEqual(a.create, "created")
         
         b = s.create_object("B", path=domain_id + "/b", included_types=["/people/person"])
-        print s.cookiejar._cookies["sandbox-freebase.com"]
         
         q = { "id" : b.id, "type" : [{"id" : None}] }
         types = map(f, s.mqlread(q)["type"])
         self.assertEqual("/common/topic" in types, True)
         self.assertEqual("/people/person" in types, True)
         self.assertEqual("/film/actor" in types, False)
-        s.touch(), time.sleep(2), s.touch();
-        print s.cookiejar._cookies["sandbox-freebase.com"]
+        
         type_object(s, b.id, "/film/film_genre")
-        print s.cookiejar._cookies["sandbox-freebase.com"]
         
-        s.touch(), time.sleep(2), s.touch(); 
-        #s.mqlwrite({"id" : b.id, "/film/film_genre/films_in_this_genre" : {"id" : "/en/the_taking_of_pelham_1_2_3", "connect" : "insert"}})
         types = map(f, s.mqlread(q)["type"])
-        print s.cookiejar._cookies["sandbox-freebase.com"]
-        
-        s.touch(), time.sleep(2), s.touch(); 
-        print types
         
         self.assertEqual("/film/film_genre" in types, True)
         self.assertEqual("/media_common/media_genre" in types, True)