make freebase.schema more readable, moved some tests, finalizing the stuff that should actually be in the package

git-svn-id: http://freebase-python.googlecode.com/svn/trunk@178 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2

1 files changed, 206 insertions, 156 deletions

diff --git a/freebase/schema.py b/freebase/schema.py
index 62e5389..459c51a 100644
--- a/freebase/schema.py
+++ b/freebase/schema.py
@@ -2,6 +2,44 @@
 from freebase.api.session import HTTPMetawebSession
 from freebase.api.session import get_key_namespace, LITERAL_TYPE_IDS
 
+"""
+NOTE
+----
+graph is used freely in this file. Some information:
+ - It refers to an internal representation of a group of types.
+ - It resembles a mqlread result, but it is not a mqlread result
+ - It also has some helper variables like __requires and __related. 
+ - It is produced by _get_graph
+ - It can be converted into valid json (json.dumps(graph, indent=2))
+ 
+Its structure is as follows:
+
+    "type_id" : {
+      "name" : "My Type",
+      "id" : "/base_id/type_id"
+      ...
+      "__requires" : ["/base_id/type_id2"]
+      "__properties" : [
+        {
+          "id" : "/base_id/type_id/my_prop"
+          ...
+        },
+        {
+          ...
+        }
+      ]
+    },
+    "type_id2" : {...}
+    ...
+    
+"""
+
+class DelegationError(Exception):
+    """You can't set the expected_type if the expected_type of the delegated (master) is a primitive"""
+
+class CVTError(Exception):
+    """You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true."""
+
 def key_exists(s, k):
     q = {
         "id" : k,
@@ -74,7 +112,7 @@ def get_property_info(s, prop_id):
             elif res[i].has_key("value"):
                 info[i] = res[i]["value"]
             else:
-                raise Exception("There is a problem with getting the property value.")
+                raise ValueError("There is a problem with getting the property value.")
         else: info[i] = None
 
     # delete the properties that are going to be asked for in create_property
@@ -89,7 +127,7 @@ def get_property_info(s, prop_id):
     # delete other useless things...
     del res["id"]
 
-    for i in [k for k, v in res.items() if v is None]:
+    for i in [k for k, v in res.iteritems() if v is None]:
         del res[i]
 
     info["__raw"] = res
@@ -97,9 +135,17 @@ def get_property_info(s, prop_id):
 
 
 # Create Type
-def create_type(s, name, key, ns, cvt=None, tip=None, included=None, extra=None):
+def create_type(s, name, key, ns, cvt=False, tip=None, included=None, extra=None):
     if key_exists(s, ns + "/" + key ):
         return
+    
+    # assert isinstance(name, basestring) # name could be mqlish
+    assert isinstance(key, basestring)
+    assert isinstance(ns, basestring)
+    
+    assert tip is None or isinstance(tip, basestring)    
+    assert included is None or isinstance(included, (basestring, list, tuple))
+    assert extra is None or isinstance(extra, dict)
 
     q = {
         "create" : "unconditional",
@@ -143,12 +189,17 @@ def create_type(s, name, key, ns, cvt=None, tip=None, included=None, extra=None)
 # Create Property
 def create_property(s, name, key, schema, expected, unique=False, disambig=False, tip=None, extra=None):
     if key_exists(s, schema + "/" + key):
-        # this isn't (neccesarily) a problem
-        # raise Exception("The key \"%s\" already exists!" % (schema + "/" + key))
         return
 
-    # validate parameters str
-
+    
+    # validate parameters
+    # assert isinstance(name, basestring) # could be mql
+    assert isinstance(key, basestring)
+    assert isinstance(schema, basestring)
+    assert isinstance(expected, basestring)
+    assert tip is None or isinstance(tip, basestring)    
+    assert extra is None or isinstance(extra, dict)
+    
     q = {
         "create" : "unconditional",
         "type" : "/type/property",
@@ -169,10 +220,17 @@ def create_property(s, name, key, schema, expected, unique=False, disambig=False
          q['/freebase/property_hints/disambiguator'] = { "connect" : "update", "value" : True }
     if extra:
          q.update(extra)
-    #print json.dumps(q, indent=2)
     return s.mqlwrite(q, use_permission_of=schema)
 
 def delegate_property(s, p, schema, name=None, key=None, expected=None, tip=None, extra=None):
+    
+    assert isinstance(p, basestring)
+    assert isinstance(schema, basestring)
+    #assert name is None or isinstance(name, basestring)
+    assert key is None or isinstance(key, basestring)
+    assert expected is None or isinstance(expected, basestring)
+    assert tip is None or isinstance(tip, basestring)
+    
     q = {
         "id" : p,
         "type" : "/type/property",
@@ -190,7 +248,7 @@ def delegate_property(s, p, schema, name=None, key=None, expected=None, tip=None
     if r["expected_type"]["id"] in LITERAL_TYPE_IDS:
         if expected:
             if expected != r["expected_type"]["id"]:
-                raise Exception("You can't set the expected_type if the expected_type of the delegated (master) is a primitive")
+                raise DelegationError("You can't set the expected_type if the expected_type of the delegated (master) is a primitive")
         expected = r["expected_type"]["id"]
     # If the expected_type of the delegator(master) is not a primitive, the delegated's
     # expected_type can be different
@@ -233,6 +291,13 @@ def reciprocate_property(s, name, key, master, unique=False, disambig=False, tip
     So, given a master, the reciprocal's schema is the master's expected type and the reciprocal's
     expected type is the master's schema. """
 
+    # assert isinstance(name, basestring) # name could be mqlish
+    assert isinstance(key, basestring)
+    assert isinstance(master, basestring)
+    
+    assert tip is None or isinstance(tip, basestring)
+    assert extra is None or isinstance(extra, dict)
+    
 
     q = {
         "id" : master,
@@ -249,174 +314,126 @@ def reciprocate_property(s, name, key, master, unique=False, disambig=False, tip
     return create_property(s, name, key, ect, schema, unique, disambig, tip,
         extra = master)
 
-# upload / restore types
+# dump / restore types
 def dump_base(s, base_id):
-    types = map(lambda x: x["id"], s.mqlread({"id" : base_id, "/type/domain/types":[{"id" : None}]})["/type/domain/types"])
+    types = [type_object["id"] for type_object in s.mqlread({"id" : base_id, "/type/domain/types":[{"id" : None}]})["/type/domain/types"]]
     graph = _get_graph(s, types, True)
-    graph["__follow_types"] = True
-    
+        
     return graph
 
 def dump_type(s, type_id, follow_types=True):
     types = [type_id]
     graph = _get_graph(s, types, follow_types)
-    graph["__follow_types"] = follow_types
     
     return graph
     
 
-def restore(s, graph, new_location, ignore_types=None, debug=False):
+def restore(s, graph, new_location, ignore_types=None):
     follow_types = graph.get("__follow_types", True)
-    if debug: print "Following types:", follow_types
     
     # create type dependencies
-    typegraph = {}
-    for tid, idres in graph.items():
-        if not tid.startswith("__"):
-            typegraph[tid] = idres["__requires"]
-    
-    type_deps = map(lambda (name, x): (len(x), name), typegraph.iteritems())
-    type_deps.sort()
-    if follow_types:
-        types_to_create = create_what(type_deps, typegraph)
-    else:
-        types_to_create = typegraph.keys()
+    type_requires_graph = {}
     
-    # create property dependencies
-    propgraph = {}
-    proptotype = {}
-    for tid, idres in graph.items():
-        if not tid.startswith("__"):
-            for prop in idres["properties"]:
-                propgraph[prop["id"]] = prop["__requires"]
-                proptotype[prop["id"]] = tid
-    prop_deps = map(lambda (name, x): (len(x), name), propgraph.items()) #*
-    prop_deps.sort()
-    if follow_types:
-        props_to_create = create_what(prop_deps, propgraph)
-    else:
-        props_to_create = propgraph.keys()
+    # create prop dependencies
+    prop_requires_graph = {}
+    prop_to_type_map = {}
+        
+    for type_id, type_information in graph.iteritems():
+        if not type_id.startswith("__"): # not a real type, but rather a helper
+            # type dependency generation
+            type_requires_graph[type_id] = type_information["__requires"]
+            
+            # prop dependency generation
+            for prop in type_information["properties"]:
+                prop_requires_graph[prop["id"]] = prop["__requires"]
+                prop_to_type_map[prop["id"]] = type_id
+            
     
-    if debug: print "types", types_to_create
-    if debug: print "-----------------------"
-    if debug: print "props", props_to_create
     
-    base_id, domain_id = s.mqlreadmulti([{"id" : types_to_create[0], "type" : "/type/type", "domain" : {"id" : None}},
-                                         {"id" : new_location, "a:id" : None}])                         
-    base_id = base_id["domain"]["id"]
-    domain_id = domain_id["a:id"]
+    types_to_create = _generate_dependency_creation_order(type_requires_graph)
+    props_to_create = _generate_dependency_creation_order(prop_requires_graph)
+        
+    origin_id, new_location_id = s.mqlreadmulti([{"id" : types_to_create[0], "type" : "/type/type", "domain" : {"id" : None}},
+                                               {"id" : new_location, "a:id" : None}])                         
+    origin_id = origin_id["domain"]["id"]
+    new_location_id = new_location_id["a:id"]
     
     only_include = types_to_create + props_to_create
     
-    for type in types_to_create:
-        if debug: print type
-        key = ""
-        if len(graph[type]["key"]) == 1:
-            key = graph[type]["key"][0]["value"]
-        else:
-            expectedname = graph[type]["id"].split("/")[-1]
-            if base_id:
-                for group in graph[type]["key"]:
-                    if group["namespace"] == base_id:
-                        key = group["value"]
-                        continue
-            if key is None:
-                key = expectedname
+    for type_id in types_to_create:
+        # let's find the type's key
+        key = None
+        for group in graph[type_id]["key"]:
+            if group["namespace"] == origin_id:
+                key = group["value"]
+                break
+        
+        if key is None: # this shouldn't happen
+            key = graph[type_id]["id"].split("/")[-1] # this can be wrong... different key than typeid
+            
+        
         tip = None
-        if graph[type]["/freebase/documented_object/tip"]:
-            tip = graph[type]["/freebase/documented_object/tip"]["value"]
+        if graph[type_id]["/freebase/documented_object/tip"]:
+            tip = graph[type_id]["/freebase/documented_object/tip"]["value"]
         
         ignore = ("name", "domain", "key", "type", "id", "properties", "/freebase/type_hints/enumeration",
                     "/freebase/type_hints/included_types", "/freebase/type_hints/mediator", "/freebase/documented_object/tip")
-        extra = {}
-        for k, v in graph[type].items():
-            if k not in ignore and not k.startswith("__"):
-                if v:
-                    if isinstance(v, basestring):
-                        extra.update({k:v})
-                    elif isinstance(v, bool):
-                        extra.update({k:v})
-                    elif v.has_key("id"):
-                        extra.update({k:v["id"]})
-                    elif v.has_key("value"):
-                        extra.update({k:v["value"]})
-                    else:
-                        raise Exception("There is a problem with getting the property value.")
+        extra = _generate_extra_properties(graph[type_id], ignore)
         
-        create_type(s, graph[type]["name"]["value"], key, domain_id,
-            included=map(lambda x: convert_name(x["id"], base_id, domain_id, only_include), graph[type]["/freebase/type_hints/included_types"]),
-            cvt=graph[type]["/freebase/type_hints/mediator"],
-            tip=tip, extra=extra)
-
-    
-    if debug: print "--------------------------"
+        name = graph[type_id]["name"]["value"]
+        included = [_convert_name_to_new(included_type["id"], origin_id, new_location_id, only_include) for included_type in graph[type_id]["/freebase/type_hints/included_types"]]
+        cvt = graph[type_id]["/freebase/type_hints/mediator"]
+        
+        create_type(s, name, key, new_location_id, included=included, cvt=cvt, tip=tip, extra=extra)
     
-    for prop in props_to_create: #* prop_id
-        info = graph[proptotype[prop]]["properties"]
-        for i in info:
-            if i["id"] == prop: 
+    for prop_id in props_to_create: #* prop_id
+        type_id = prop_to_type_map[prop_id]
+        all_properties_for_type = graph[type_id]["properties"]
+        for prop in all_properties_for_type:
+            if prop["id"] == prop_id: # good, we are dealing with our specific property 
+                
+                new_schema = _convert_name_to_new(type_id, origin_id, new_location_id, only_include)
+                
+                name =  prop["name"]
                 
-                schema = convert_name(proptotype[prop], base_id, domain_id, only_include)
-                if debug: print prop
                 expected = None
+                if prop["expected_type"]:
+                    expected = _convert_name_to_new(prop["expected_type"], origin_id, new_location_id, only_include)
                 
-                if i["expected_type"]:
-                    expected = convert_name(i["expected_type"], base_id, domain_id, only_include)
-                for k in i["key"]:
-                    if k["namespace"] == proptotype[prop]:
-                        key = k["value"]
-                if i["/freebase/documented_object/tip"]:
-                    tip = graph[type]["/freebase/documented_object/tip"]
+                for group in prop["key"]:
+                    if group["namespace"] == type_id:
+                        key = group["value"]
+                        break
+                
+                tip = None
+                if prop["/freebase/documented_object/tip"]:
+                    tip = prop["/freebase/documented_object/tip"]["value"]
                 
-                disambig = i["/freebase/property_hints/disambiguator"]
+                disambig = prop["/freebase/property_hints/disambiguator"]
+                unique = prop["unique"]
                 
                 ignore = ("name", "expected_type", "key", "id", "master_property", "delegated", "unique", "type", "schema",
                             "/freebase/property_hints/disambiguator", "enumeration", "/freebase/property_hints/enumeration", 
                             "/freebase/documented_object/tip")
-                extra = {}
-                for k, v in i.items():
-                    if k not in ignore and not k.startswith("__"):
-                        if v:
-                            if isinstance(v, basestring):
-                                extra.update({k:v})
-                            elif isinstance(v, bool):
-                                extra.update({k:v})
-                            elif v.has_key("id"):
-                                extra.update({k:v["id"]})
-                            elif v.has_key("value"):
-                                extra.update({k:v["value"]})
-                            else:
-                                raise Exception("There is a problem with getting the property value.")
-                            
-                            # since we are creating a property, all these connect insert delicacies are unneccesary    
-                            """if isinstance(v, basestring) and v.startswith("/"): # an id
-                                extra.update({k : {"connect" : "insert", "id" : v}})
-                            elif isinstance(v, bool): # a bool value
-                                extra.update({k : {"connect" : "insert", "value" : v}})
-                            else: # an english value
-                                extra.update({k : {"connect" : "insert", "value" : v, "lang" : "/lang/en"}})"""
                 
+                extra = _generate_extra_properties(prop, ignore)   
                 
-                if i['master_property']:
-                    converted_master_property = convert_name(i["master_property"], base_id, domain_id, only_include)
-                    if converted_master_property == i["master_property"]:
-                        raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
+                if prop['master_property']:
+                    converted_master_property = _convert_name_to_new(prop["master_property"], origin_id, new_location_id, only_include)
+                    if converted_master_property == prop["master_property"]:
+                        raise CVTError("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
                                         "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
-                    reciprocate_property(s, i["name"], key, converted_master_property,
-                        i["unique"], disambig=disambig, tip=tip, extra=extra)
+                    reciprocate_property(s, name, key, converted_master_property,
+                        unique, disambig=disambig, tip=tip, extra=extra)
                 
-                elif i['delegated']:
-                    delegate_property(s, convert_name(i['delegated'], base_id, domain_id, only_include), schema,
+                elif prop['delegated']:
+                    delegate_property(s, _convert_name_to_new(prop['delegated'], origin_id, new_location_id, only_include), new_schema,
                         expected=expected, tip=tip, extra=extra)
                 
                 else:
-                    create_property(s, i["name"], key, schema, expected, i["unique"], 
+                    create_property(s, name, key, new_schema, expected, unique, 
                         disambig=disambig, tip=tip, extra=extra) 
                         
-                
-        
-    
-
 
 def _get_graph(s, initial_types, follow_types):
     """ get the graph of dependencies of all the types involved, starting with a list supplied """
@@ -434,40 +451,73 @@ def _get_graph(s, initial_types, follow_types):
             done.update(graph[new]["__related"])
         if not follow_types:
             # we have to check that there are no cvts attached to us, or else
-            # ugly things happen (we can't include the cvt)
+            # ugly things happen (we can't include the cvt because the cvt won't link to us.)
             for prop in graph[new]["properties"]:
                 if prop["master_property"]:
-                    raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
+                    raise CVTError("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
                                     "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
+    
+    graph["__follow_types"] = follow_types
     return graph
 
 
-def convert_name(old_name, operating_base, new_base, only_include=None):
+def _convert_name_to_new(old_name, operating_base, new_base, only_include=None):
     if old_name in only_include and old_name.startswith(operating_base):
         return new_base + old_name.replace(operating_base, "", 1)
     else:
         return old_name
 
-def create_what(deps, graph):
-    create_list = []
-    while len(deps) > 0:
-        neediness, id = deps.pop(0)
-        if neediness == 0:
-            create_list.append(id)
+def _generate_dependency_creation_order(requires_graph):
+    # This is a naive topographical sort to determine
+    # in what order to create types or properties so
+    # that the other type/properties they rely on
+    # are already created
+    
+    # This function is called with the type dependencies
+    # and then the property dependencies.
+    
+    
+    # we sort the dependency_list because its a good idea
+    # to create the guys with zero dependencies before the
+    # guys with one.. it's just a simple optimization to 
+    # the topographical sort
+    dependency_list = [(len(requires), name) for (name, requires) in requires_graph.iteritems()]
+    dependency_list.sort()
+    
+    creation_order_list = []
+    while len(dependency_list) > 0:
+        number_of_requirements, id = dependency_list.pop(0)
+        if number_of_requirements == 0:
+            creation_order_list.append(id)
             continue
         else:
-            work = True
-            for req in graph[id]:
-                if req not in create_list:
-                    work = False
+            are_the_types_dependencies_already_resolved = True
+            for requirement in requires_graph[id]:
+                if requirement not in creation_order_list:
+                    are_the_types_dependencies_already_resolved = False
                     continue
-            if work:
-                create_list.append(id)
+            if are_the_types_dependencies_already_resolved:
+                creation_order_list.append(id)
             else:
-                deps.append((neediness, id))
-    return create_list        
-
+                dependency_list.append((number_of_requirements, id))
+    return creation_order_list
 
+def _generate_extra_properties(dictionary_of_values, ignore):
+    extra = {}
+    for k, v in dictionary_of_values.iteritems():
+        if k not in ignore and not k.startswith("__"):
+            if v:
+                if isinstance(v, basestring):
+                    extra.update({k:v})
+                elif isinstance(v, bool):
+                    extra.update({k:v})
+                elif v.has_key("id"):
+                    extra.update({k:v["id"]})
+                elif v.has_key("value"):
+                    extra.update({k:v["value"]})
+                else:
+                    raise ValueError("There is a problem with getting the property value.")
+    return extra
 
 def _get_needed(s, type_id):
     q = TYPE_QUERY
@@ -491,8 +541,8 @@ def _get_needed(s, type_id):
     # related can be used to generate subgraphs of types
     # required is used to generate the dependency graph of types
 
-    related = return_relevant(related_types, parents)
-    requires = return_relevant(included_types, parents)
+    related = _return_relevant(related_types, parents)
+    requires = _return_relevant(included_types, parents)
 
     # get property information
     properties = r["properties"]
@@ -503,7 +553,7 @@ def _get_needed(s, type_id):
         if prop["delegated"]:
             dependent_on.add(prop["delegated"])
 
-        prop["__requires"] = return_relevant(dependent_on, parents)
+        prop["__requires"] = _return_relevant(dependent_on, parents)
 
     # return all the information along with our special __* properties
     info = r
@@ -512,7 +562,7 @@ def _get_needed(s, type_id):
     return info
 
 
-def return_relevant(start_list, parents):
+def _return_relevant(start_list, parents):
     final = []
     for item in start_list:
         indomain = False