add upload/revert schema manipulation, more tests, and other changes of the past few days

git-svn-id: http://freebase-python.googlecode.com/svn/trunk@172 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2

1 files changed, 571 insertions, 0 deletions

diff --git a/freebase/schema.py b/freebase/schema.py
new file mode 100644
index 0000000..5be9b4a
--- /dev/null
+++ b/freebase/schema.py
@@ -0,0 +1,571 @@
+
+from freebase.api.session import HTTPMetawebSession
+from freebase.api.session import get_key_namespace, LITERAL_TYPE_IDS
+
+def key_exists(s, k):
+    q = {
+        "id" : k,
+        "guid" : None
+    }
+    return not None == s.mqlread(q)
+
+
+def type_object(s, id, type_id):
+    q = {
+        "id" : type_id,
+        "/freebase/type_hints/included_types" : [{"id" : None}]
+    }
+    included_types = map(lambda x: x["id"], s.mqlread(q)["/freebase/type_hints/included_types"])
+    
+    wq = {
+        "id" : id,
+        "type" : [{
+            "id" : it,
+            "connect" : "insert"
+        } for it in included_types + [type_id]]
+    }
+    return s.mqlwrite(wq)
+
+
+def copy_property(s, id, newid, **extra):
+    newname, newschema = get_key_namespace(newid)
+    
+    info = get_property_info(s, id)
+    info["__raw"].update(extra)
+    create_property(s, info["name"], newname, newschema, info["expected_type"], info["unique"], info["/freebase/property_hints/disambiguator"],
+        info["/freebase/documented_object/tip"], info["__raw"])
+
+def move_property(s, id, newid, **extra):
+    copy_property(s, id, newid, **extra)
+    disconnect_schema = {"type" : "/type/property", "schema" : {"connect" : "delete", "id" : "/".join(id.split("/")[:-1]) }}    
+    s.disconnect_object(id, extra = disconnect_schema)
+
+def get_property_info(s, prop_id):
+    q = PROPERTY_QUERY
+    q.update(id=prop_id)
+    res = s.mqlread(q)
+    info = {}
+    
+    info["name"] = res["name"]["value"]
+    if res["schema"]:
+        info["schema"] = res["schema"]["id"]
+    else: info["schema"] = None
+
+    if res["key"]:
+        info["keys"] = map(lambda x: (x["value"], x["namespace"]), res["key"])
+    else: info["key"] = None
+
+    if res["/freebase/documented_object/tip"]:
+        info["/freebase/documented_object/tip"] = res["/freebase/documented_object/tip"]["value"]
+    else: info["/freebase/documented_object/tip"] = None
+
+    for i in ["delegated", "enumeration", "expected_type", "id", "master_property", "unique", "unit",
+     "/freebase/property_hints/disambiguator", "/freebase/property_hints/display_none",
+     "/freebase/property_hints/display_orientation","/freebase/property_hints/enumeration",
+      "/freebase/property_hints/dont_display_in_weblinks", "/freebase/property_hints/inverse_description"]:
+
+        if res[i]:
+            if isinstance(res[i], basestring):
+                info[i] = res[i]
+            elif isinstance(res[i], bool):
+                info[i] = res[i]
+            elif res[i].has_key("id"):
+                info[i] = res[i]["id"]
+            elif res[i].has_key("value"):
+                info[i] = res[i]["value"]
+            else:
+                raise Exception("There is a problem with getting the property value.")
+        else: info[i] = None
+
+    # delete the properties that are going to be asked for in create_property
+    del res["name"]
+    del res["schema"]
+    del res["key"]
+    del res["expected_type"]
+    del res["unique"]
+    del res["/freebase/property_hints/disambiguator"]
+    del res["/freebase/documented_object/tip"]
+
+    # delete other useless things...
+    del res["id"]
+
+    for i in [k for k, v in res.items() if v is None]:
+        del res[i]
+
+    info["__raw"] = res
+    return info
+
+
+# Create Type
+def create_type(s, name, key, ns, cvt=None, tip=None, included=None, extra=None):
+    if key_exists(s, ns + "/" + key ):
+        return
+
+    q = {
+        "create" : "unconditional",
+        "type" : "/type/type",
+        "/type/type/domain" : { "connect" : "insert", "id" : ns },
+        "name" : {"connect" : "insert", "value" : name, "lang" : "/lang/en" },
+        "key" : {
+            "connect" : "insert",
+            "value" : key,
+            "namespace" : ns
+        }
+    }
+
+    if included:
+        if isinstance(included, basestring):
+            included = [included]
+        itsq = [{
+            "id|=" : included,
+            "/freebase/type_hints/included_types" : [{"id" : None}]
+        }]
+        r = s.mqlread(itsq)
+        included_types = set(included)
+        if r:
+            for i in r:
+                included_types.update(map(lambda x: x["id"], i["/freebase/type_hints/included_types"]))
+
+        its = [{"connect" : "insert", "id" : t} for t in included_types]
+        q['/freebase/type_hints/included_types'] = its
+
+    # TODO: enum
+
+    if cvt:
+        q['/freebase/type_hints/mediator'] = { "connect" : "update", "value" : True }
+    if tip:
+        q['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" }
+    
+    if extra: q.update(extra)
+    return s.mqlwrite(q, use_permission_of=ns)
+
+
+# Create Property
+def create_property(s, name, key, schema, expected, unique=False, disambig=False, tip=None, extra=None):
+    if key_exists(s, schema + "/" + key):
+        raise Exception("The key \"%s\" already exists!" % (schema + "/" + key))
+        return
+
+    # validate parameters str
+
+    q = {
+        "create" : "unconditional",
+        "type" : "/type/property",
+        "name" : name,
+        "key" : {
+            "connect" : "insert",
+            "value" : key,
+            "namespace" : { "id" : schema },
+        },
+        "schema" : { "connect" : "insert", "id" : schema },
+        "expected_type" : { "connect" : "insert", "id" : expected }
+    }
+    if unique:
+         q['unique'] = { "connect" : "update", "value" : unique }
+    if tip:
+         q['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" }
+    if disambig:
+         q['/freebase/property_hints/disambiguator'] = { "connect" : "update", "value" : True }
+    if extra:
+         q.update(extra)
+    #print json.dumps(q, indent=2)
+    return s.mqlwrite(q, use_permission_of=schema)
+
+def delegate_property(s, p, schema, name=None, key=None, expected=None, tip=None, extra=None):
+    q = {
+        "id" : p,
+        "type" : "/type/property",
+        "name" : None,
+        "unique" : None,
+        "expected_type" : {"id" : None},
+        "key" : None,
+        "/freebase/documented_object/tip" : None,
+        "/freebase/property_hints/disambiguator" : None
+    }
+    r = s.mqlread(q)
+
+    # If the expected_type of the delegator(master) is a primitive, the delegated's
+    # expected_type must be the same
+    if r["expected_type"]["id"] in LITERAL_TYPE_IDS:
+        if expected:
+            if expected != r["expected_type"]["id"]:
+                raise Exception("You can't set the expected_type if the expected_type of the delegated (master) is a primitive")
+        expected = r["expected_type"]["id"]
+    # If the expected_type of the delegator(master) is not a primitive, the delegated's
+    # expected_type can be different
+    elif expected is None:
+        expected = r["expected_type"]["id"]
+
+
+    if not tip and r["/freebase/documented_object/tip"]:
+        tip = r["/freebase/documented_object/tip"]
+
+    if name is None:
+        name = r["name"]
+    if key is None:
+        key = r["key"]
+    
+    delegate = { "/type/property/delegated" : p}
+    if extra: delegate.update(extra)
+    
+    return create_property(s, name, key, schema, expected, r['unique'],
+        r["/freebase/property_hints/disambiguator"],
+        tip,
+        delegate)
+
+def reciprocate_property(s, name, key, master, unique=False, disambig=False, tip=None, extra=None):
+    """ We're creating a reciprocate property of the master property. Let's illustrate
+    the idea behind the function with an example.
+
+    Say we examine the /visual_art/art_period_movement/associated_artworks property.
+    An example of an art_period_movement is the Renaissance, and once associated_artworks
+    could be the /en/mona_lisa. In this example, /visual_art/art_period_movement/associated_artworks
+    will be the master property, and /visual_art/artwork/period_or_movement will be the reciprocal.
+
+    In order to determine the characterists of the reciprocal property, we must examine the master.
+    associated_artworks property's schema is /visual_art/art_period_movement and its expected
+    type is /visual_art/artwork. Notice the similarity to /visual_art/artwork/period_or_movement.
+    period_or_movement's schema is /visual_art/artwork -- art_period_movement's expected type.
+    period_or_movement's expected type is /visual_art/art_period_movement -- art_period_movement's
+    schema!
+
+    So, given a master, the reciprocal's schema is the master's expected type and the reciprocal's
+    expected type is the master's schema. """
+
+
+    q = {
+        "id" : master,
+        "/type/property/expected_type" : None,
+        "/type/property/schema" : None }
+    r = s.mqlread(q)
+    ect = r["/type/property/expected_type"]
+    schema = r["/type/property/schema"]
+
+    master = {"master_property" : master}
+    if extra: master.update(extra)
+
+    # NOTE: swapping ect and schema; see comment above
+    return create_property(s, name, key, ect, schema, unique, disambig, tip,
+        extra = master)
+
+# upload / restore types
+def dump_base(s, base_id):
+    types = map(lambda x: x["id"], s.mqlread({"id" : base_id, "/type/domain/types":[{"id" : None}]})["/type/domain/types"])
+    graph = _get_graph(types)
+    graph["__follow_types"] = True
+    
+    return graph
+
+def dump_type(s, type_id, follow_types=True):
+    types = [type_id]
+    graph = _get_graph(types, follow_types)
+    graph["__follow_types"] = follow_types
+    result = json.dumps(graph, indent=2)
+    
+    fh = open("junk.json", "w")
+    fh.write(result)
+    fh.close()
+    
+    return graph
+    
+
+def upload_type(s, graph, new_location, ignore_types=None, debug=False):
+    follow_types = graph.get("__follow_types", True)
+    if debug: print "Following types:", follow_types
+    
+    # create type dependencies
+    typegraph = {}
+    for tid, idres in graph.items():
+        if not tid.startswith("__"):
+            typegraph[tid] = idres["__requires"]
+    
+    type_deps = map(lambda (name, x): (len(x), name), typegraph.items())
+    type_deps.sort()
+    if follow_types:
+        types_to_create = create_what(type_deps, typegraph)
+    else:
+        types_to_create = typegraph.keys()
+    
+    # create property dependencies
+    propgraph = {}
+    proptotype = {}
+    for tid, idres in graph.items():
+        if not tid.startswith("__"):
+            for prop in idres["properties"]:
+                propgraph[prop["id"]] = prop["__requires"]
+                proptotype[prop["id"]] = tid
+    prop_deps = map(lambda (name, x): (len(x), name), propgraph.items())
+    prop_deps.sort()
+    if follow_types:
+        props_to_create = create_what(prop_deps, propgraph)
+    else:
+        props_to_create = propgraph.keys()
+    
+    if debug: print "types", types_to_create
+    if debug: print "-----------------------"
+    if debug: print "props", props_to_create
+    
+    base_id, domain_id = s.mqlreadmulti([{"id" : types_to_create[0], "type" : "/type/type", "domain" : {"id" : None}},
+                                         {"id" : new_location, "a:id" : None}])                         
+    base_id = base_id["domain"]["id"]
+    domain_id = domain_id["a:id"]
+    
+    only_include = types_to_create + props_to_create
+    
+    for type in types_to_create:
+        if debug: print type
+        key = ""
+        if len(graph[type]["key"]) == 1:
+            key = graph[type]["key"][0]["value"]
+        else:
+            expectedname = graph[type]["id"].split("/")[-1]
+            if base_id:
+                for group in graph[type]["key"]:
+                    if group["namespace"] == base_id:
+                        key = group["value"]
+                        continue
+            if key is None:
+                key = expectedname
+        tip = None
+        if graph[type]["/freebase/documented_object/tip"]:
+            tip = graph[type]["/freebase/documented_object/tip"]["value"]
+        
+        ignore = ("name", "domain", "key", "type", "id", "properties", "/freebase/type_hints/enumeration",
+                    "/freebase/type_hints/included_types", "/freebase/type_hints/mediator", "/freebase/documented_object/tip")
+        extra = {}
+        for k, v in graph[type].items():
+            if k not in ignore and not k.startswith("__"):
+                if v:
+                    if isinstance(v, basestring):
+                        extra.update({k:v})
+                    elif isinstance(v, bool):
+                        extra.update({k:v})
+                    elif v.has_key("id"):
+                        extra.update({k:v["id"]})
+                    elif v.has_key("value"):
+                        extra.update({k:v["value"]})
+                    else:
+                        raise Exception("There is a problem with getting the property value.")
+        
+        create_type(s, graph[type]["name"]["value"], key, domain_id,
+            included=map(lambda x: convert_name(x["id"], base_id, domain_id, only_include), graph[type]["/freebase/type_hints/included_types"]),
+            cvt=graph[type]["/freebase/type_hints/mediator"],
+            tip=tip, extra=extra)
+
+    
+    if debug: print "--------------------------"
+    
+    for prop in props_to_create:
+        info = graph[proptotype[prop]]["properties"]
+        for i in info:
+            if i["id"] == prop: 
+                
+                schema = convert_name(proptotype[prop], base_id, domain_id, only_include)
+                if debug: print prop
+                expected = None
+                
+                if i["expected_type"]:
+                    expected = convert_name(i["expected_type"], base_id, domain_id, only_include)
+                for k in i["key"]:
+                    if k.namespace == proptotype[prop]:
+                        key = k.value
+                if i["/freebase/documented_object/tip"]:
+                    tip = graph[type]["/freebase/documented_object/tip"]
+                
+                disambig = i["/freebase/property_hints/disambiguator"]
+                
+                ignore = ("name", "expected_type", "key", "id", "master_property", "delegated", "unique", "type", "schema",
+                            "/freebase/property_hints/disambiguator", "enumeration", "/freebase/property_hints/enumeration", 
+                            "/freebase/documented_object/tip")
+                extra = {}
+                for k, v in i.items():
+                    if k not in ignore and not k.startswith("__"):
+                        if v:
+                            if isinstance(v, basestring):
+                                extra.update({k:v})
+                            elif isinstance(v, bool):
+                                extra.update({k:v})
+                            elif v.has_key("id"):
+                                extra.update({k:v["id"]})
+                            elif v.has_key("value"):
+                                extra.update({k:v["value"]})
+                            else:
+                                raise Exception("There is a problem with getting the property value.")
+                            
+                            # since we are creating a property, all these connect insert delicacies are unneccesary    
+                            """if isinstance(v, basestring) and v.startswith("/"): # an id
+                                extra.update({k : {"connect" : "insert", "id" : v}})
+                            elif isinstance(v, bool): # a bool value
+                                extra.update({k : {"connect" : "insert", "value" : v}})
+                            else: # an english value
+                                extra.update({k : {"connect" : "insert", "value" : v, "lang" : "/lang/en"}})"""
+                
+                
+                if i['master_property']:
+                    converted_master_property = convert_name(i["master_property"], base_id, domain_id, only_include)
+                    if converted_master_property == i["master_property"]:
+                        raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
+                                        "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
+                    reciprocate_property(s, i["name"], key, converted_master_property,
+                        i["unique"], disambig=disambig, tip=tip, extra=extra)
+                
+                elif i['delegated']:
+                    delegate_property(s, convert_name(i['delegated'], base_id, domain_id, only_include), schema,
+                        expected=expected, tip=tip, extra=extra)
+                
+                else:
+                    create_property(s, i["name"], key, schema, expected, i["unique"], 
+                        disambig=disambig, tip=tip, extra=extra) 
+                        
+                
+        
+    
+
+
+def _get_graph(initial_types, follow_types):
+    """ get the graph of dependencies of all the types involved, starting with a list supplied """
+    
+    assert isinstance(initial_types, (list, tuple))
+    
+    graph = {}
+    to_update = set(initial_types)
+    done = set()
+    while len(to_update) > 0:
+        new = to_update.pop()
+        graph[new] = _get_needed(s, new)
+        if follow_types:
+            [to_update.add(b) for b in graph[new]["__related"] if b not in done]
+            done.update(graph[new]["__related"])
+        if not follow_types:
+            # we have to check that there are no cvts attached to us, or else
+            # ugly things happen (we can't include the cvt)
+            for prop in graph[new]["properties"]:
+                if prop["master_property"]:
+                    raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
+                                    "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
+    return graph
+
+
+def convert_name(old_name, operating_base, new_base, only_include=None):
+    if old_name in only_include and old_name.startswith(operating_base):
+        return new_base + old_name.replace(operating_base, "", 1)
+    else:
+        return old_name
+
+def create_what(deps, graph):
+    create_list = []
+    while len(deps) > 0:
+        neediness, id = deps.pop(0)
+        if neediness == 0:
+            create_list.append(id)
+            continue
+        else:
+            work = True
+            for req in graph[id]:
+                if req not in create_list:
+                    work = False
+                    continue
+            if work:
+                create_list.append(id)
+            else:
+                deps.append((neediness, id))
+    return create_list        
+
+
+
+def _get_needed(s, type_id):
+    q = TYPE_QUERY
+    q.update(id=type_id)
+
+    r = s.mqlread(q)
+    properties = r.properties
+
+    # let's identify who the parent is in order to only include
+    # other types in that domain. We don't want to go around including
+    # all of commons because someone's a /people/person
+    parents = [r["domain"]["id"]]
+
+    included_types = map(lambda x: x["id"], r["/freebase/type_hints/included_types"])
+    related_types = set(included_types)
+    for prop in properties:
+        if prop["expected_type"]:
+            related_types.add(prop["expected_type"])
+
+    # we have two different types of relationships: required and related.
+    # related can be used to generate subgraphs of types
+    # required is used to generate the dependency graph of types
+
+    related = return_relevant(related_types, parents)
+    requires = return_relevant(included_types, parents)
+
+    # get property information
+    properties = r["properties"]
+    for prop in properties:
+        dependent_on = set()
+        if prop["master_property"]:
+            dependent_on.add(prop["master_property"])
+        if prop["delegated"]:
+            dependent_on.add(prop["delegated"])
+
+        prop["__requires"] = return_relevant(dependent_on, parents)
+
+    # return all the information along with our special __* properties
+    info = r
+    info.update(__related=related, __requires=requires, __properties=properties)
+
+    return info
+
+
+def return_relevant(start_list, parents):
+    final = []
+    for item in start_list:
+        indomain = False
+        for parent in parents:
+            if item.startswith(parent):
+                indomain = True
+                continue
+        if indomain:
+            final.append(item)
+    return final
+
+
+
+PROPERTY_QUERY = {
+        "optional" : True,
+        "type" : "/type/property",
+        "delegated" : None,
+        "enumeration" : None, 
+        "expected_type" : None,
+        "id" : None,
+        "key" : [{
+            "namespace" : None, 
+            "value" : None
+        }],
+        #"link" : [{}], 
+        "master_property" : None,
+        "name" : {"value" : None, "lang" : "/lang/en", "optional":True}, 
+        "schema" : {"id" : None, "name" : None},
+        "unique" : None, 
+        "unit" : None,
+        "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional" : True},
+        "/freebase/property_hints/disambiguator" : None,
+        "/freebase/property_hints/display_none" : None,
+        "/freebase/property_hints/display_orientation" : None,
+        "/freebase/property_hints/enumeration" : None,
+        "/freebase/property_hints/dont_display_in_weblinks" : None,
+        "/freebase/property_hints/inverse_description" : None,
+    }
+
+TYPE_QUERY = {
+        "type" : "/type/type",
+        "domain" : {},
+        "key" : [{"namespace" : None, "value" : None}],
+        "name" : {"value" : None, "lang" : "/lang/en", "optional":True},
+        "/freebase/type_hints/included_types" : [{"id" : None, "optional" : True}],
+        "/freebase/type_hints/mediator" : None,
+        "/freebase/type_hints/enumeration" : None,
+        "/freebase/type_hints/minor" : None,
+        "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional":True},
+        }
+TYPE_QUERY.update(properties=[PROPERTY_QUERY])    
+