path: root/freebase/schema.py

from freebase.api.session import HTTPMetawebSession
from freebase.api.session import get_key_namespace, LITERAL_TYPE_IDS

"""
NOTE
----
graph is used freely in this file. Some information:
 - It refers to an internal representation of a group of types.
 - It resembles a mqlread result, but it is not a mqlread result
 - It also has some helper variables like __requires and __related. 
 - It is produced by _get_graph
 - It can be converted into valid json (json.dumps(graph, indent=2))
 
Its structure is as follows:

    "type_id" : {
      "name" : "My Type",
      "id" : "/base_id/type_id"
      ...
      "__requires" : ["/base_id/type_id2"]
      "__properties" : [
        {
          "id" : "/base_id/type_id/my_prop"
          ...
        },
        {
          ...
        }
      ]
    },
    "type_id2" : {...}
    ...
    
"""

class DelegationError(Exception):
    """You can't set the expected_type if the expected_type of the delegated (master) is a primitive"""

class CVTError(Exception):
    """You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true."""

def key_exists(s, k):
    q = {
        "id" : k,
        "guid" : None
    }
    return not None == s.mqlread(q)


def type_object(s, id, type_id):
    q = {
        "id" : type_id,
        "/freebase/type_hints/included_types" : [{"id" : None}]
    }
    included_types = map(lambda x: x["id"], s.mqlread(q)["/freebase/type_hints/included_types"])
    
    wq = {
        "id" : id,
        "type" : [{
            "id" : it,
            "connect" : "insert"
        } for it in included_types + [type_id]]
    }
    return s.mqlwrite(wq)


def copy_property(s, id, newid, **extra):
    newname, newschema = get_key_namespace(newid)
    
    info = get_property_info(s, id)
    info["__raw"].update(extra)
    create_property(s, info["name"], newname, newschema, info["expected_type"], info["unique"], info["/freebase/property_hints/disambiguator"],
        info["/freebase/documented_object/tip"], info["__raw"])

def move_property(s, id, newid, **extra):
    copy_property(s, id, newid, **extra)
    disconnect_schema = {"type" : "/type/property", "schema" : {"connect" : "delete", "id" : "/".join(id.split("/")[:-1]) }}    
    s.disconnect_object(id, extra = disconnect_schema)

def get_property_info(s, prop_id):
    q = PROPERTY_QUERY
    q.update(id=prop_id)
    res = s.mqlread(q)
    info = {}
    
    info["name"] = res["name"]["value"]
    if res["schema"]:
        info["schema"] = res["schema"]["id"]
    else: info["schema"] = None

    if res["key"]:
        info["keys"] = map(lambda x: (x["value"], x["namespace"]), res["key"])
    else: info["key"] = None

    if res["/freebase/documented_object/tip"]:
        info["/freebase/documented_object/tip"] = res["/freebase/documented_object/tip"]["value"]
    else: info["/freebase/documented_object/tip"] = None

    for i in ["delegated", "enumeration", "expected_type", "id", "master_property", "unique", "unit",
     "/freebase/property_hints/disambiguator", "/freebase/property_hints/display_none",
     "/freebase/property_hints/display_orientation","/freebase/property_hints/enumeration",
      "/freebase/property_hints/dont_display_in_weblinks", "/freebase/property_hints/inverse_description"]:

        if res[i]:
            if isinstance(res[i], basestring):
                info[i] = res[i]
            elif isinstance(res[i], bool):
                info[i] = res[i]
            elif res[i].has_key("id"):
                info[i] = res[i]["id"]
            elif res[i].has_key("value"):
                info[i] = res[i]["value"]
            else:
                raise ValueError("There is a problem with getting the property value.")
        else: info[i] = None

    # delete the properties that are going to be asked for in create_property
    del res["name"]
    del res["schema"]
    del res["key"]
    del res["expected_type"]
    del res["unique"]
    del res["/freebase/property_hints/disambiguator"]
    del res["/freebase/documented_object/tip"]

    # delete other useless things...
    del res["id"]

    for i in [k for k, v in res.iteritems() if v is None]:
        del res[i]

    info["__raw"] = res
    return info


# Create Type
def create_type(s, name, key, ns, cvt=False, tip=None, included=None, extra=None):
    if key_exists(s, ns + "/" + key ):
        return
    
    # assert isinstance(name, basestring) # name could be mqlish
    assert isinstance(key, basestring)
    assert isinstance(ns, basestring)
    
    assert tip is None or isinstance(tip, basestring)    
    assert included is None or isinstance(included, (basestring, list, tuple))
    assert extra is None or isinstance(extra, dict)

    q = {
        "create" : "unconditional",
        "type" : "/type/type",
        "/type/type/domain" : { "connect" : "insert", "id" : ns },
        "name" : {"connect" : "insert", "value" : name, "lang" : "/lang/en" },
        "key" : {
            "connect" : "insert",
            "value" : key,
            "namespace" : ns
        }
    }

    if included:
        if isinstance(included, basestring):
            included = [included]
        itsq = [{
            "id|=" : included,
            "/freebase/type_hints/included_types" : [{"id" : None}]
        }]
        r = s.mqlread(itsq)
        included_types = set(included)
        if r:
            for i in r:
                included_types.update(map(lambda x: x["id"], i["/freebase/type_hints/included_types"]))

        its = [{"connect" : "insert", "id" : t} for t in included_types]
        q['/freebase/type_hints/included_types'] = its

    # TODO: enum

    if cvt:
        q['/freebase/type_hints/mediator'] = { "connect" : "update", "value" : True }
    if tip:
        q['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" }
    
    if extra: q.update(extra)
    return s.mqlwrite(q, use_permission_of=ns)


# Create Property
def create_property(s, name, key, schema, expected, unique=False, disambig=False, tip=None, extra=None):
    if key_exists(s, schema + "/" + key):
        return

    
    # validate parameters
    # assert isinstance(name, basestring) # could be mql
    assert isinstance(key, basestring)
    assert isinstance(schema, basestring)
    assert isinstance(expected, basestring)
    assert tip is None or isinstance(tip, basestring)    
    assert extra is None or isinstance(extra, dict)
    
    q = {
        "create" : "unconditional",
        "type" : "/type/property",
        "name" : name,
        "key" : {
            "connect" : "insert",
            "value" : key,
            "namespace" : { "id" : schema },
        },
        "schema" : { "connect" : "insert", "id" : schema },
        "expected_type" : { "connect" : "insert", "id" : expected }
    }
    if unique:
         q['unique'] = { "connect" : "update", "value" : unique }
    if tip:
         q['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" }
    if disambig:
         q['/freebase/property_hints/disambiguator'] = { "connect" : "update", "value" : True }
    if extra:
         q.update(extra)
    return s.mqlwrite(q, use_permission_of=schema)

def delegate_property(s, p, schema, name=None, key=None, expected=None, tip=None, extra=None):
    
    assert isinstance(p, basestring)
    assert isinstance(schema, basestring)
    #assert name is None or isinstance(name, basestring)
    assert key is None or isinstance(key, basestring)
    assert expected is None or isinstance(expected, basestring)
    assert tip is None or isinstance(tip, basestring)
    
    q = {
        "id" : p,
        "type" : "/type/property",
        "name" : None,
        "unique" : None,
        "expected_type" : {"id" : None},
        "key" : None,
        "/freebase/documented_object/tip" : None,
        "/freebase/property_hints/disambiguator" : None
    }
    r = s.mqlread(q)

    # If the expected_type of the delegator(master) is a primitive, the delegated's
    # expected_type must be the same
    if r["expected_type"]["id"] in LITERAL_TYPE_IDS:
        if expected:
            if expected != r["expected_type"]["id"]:
                raise DelegationError("You can't set the expected_type if the expected_type of the delegated (master) is a primitive")
        expected = r["expected_type"]["id"]
    # If the expected_type of the delegator(master) is not a primitive, the delegated's
    # expected_type can be different
    elif expected is None:
        expected = r["expected_type"]["id"]


    if not tip and r["/freebase/documented_object/tip"]:
        tip = r["/freebase/documented_object/tip"]

    if name is None:
        name = r["name"]
    if key is None:
        key = r["key"]
    
    delegate = { "/type/property/delegated" : p}
    if extra: delegate.update(extra)
    
    return create_property(s, name, key, schema, expected, r['unique'],
        r["/freebase/property_hints/disambiguator"],
        tip,
        delegate)

def reciprocate_property(s, name, key, master, unique=False, disambig=False, tip=None, extra=None):
    """ We're creating a reciprocate property of the master property. Let's illustrate
    the idea behind the function with an example.

    Say we examine the /visual_art/art_period_movement/associated_artworks property.
    An example of an art_period_movement is the Renaissance, and once associated_artworks
    could be the /en/mona_lisa. In this example, /visual_art/art_period_movement/associated_artworks
    will be the master property, and /visual_art/artwork/period_or_movement will be the reciprocal.

    In order to determine the characterists of the reciprocal property, we must examine the master.
    associated_artworks property's schema is /visual_art/art_period_movement and its expected
    type is /visual_art/artwork. Notice the similarity to /visual_art/artwork/period_or_movement.
    period_or_movement's schema is /visual_art/artwork -- art_period_movement's expected type.
    period_or_movement's expected type is /visual_art/art_period_movement -- art_period_movement's
    schema!

    So, given a master, the reciprocal's schema is the master's expected type and the reciprocal's
    expected type is the master's schema. """

    # assert isinstance(name, basestring) # name could be mqlish
    assert isinstance(key, basestring)
    assert isinstance(master, basestring)
    
    assert tip is None or isinstance(tip, basestring)
    assert extra is None or isinstance(extra, dict)
    

    q = {
        "id" : master,
        "/type/property/expected_type" : None,
        "/type/property/schema" : None }
    r = s.mqlread(q)
    ect = r["/type/property/expected_type"]
    schema = r["/type/property/schema"]

    master = {"master_property" : master}
    if extra: master.update(extra)

    # NOTE: swapping ect and schema; see comment above
    return create_property(s, name, key, ect, schema, unique, disambig, tip,
        extra = master)

# dump / restore types
def dump_base(s, base_id):
    types = [type_object["id"] for type_object in s.mqlread({"id" : base_id, "/type/domain/types":[{"id" : None}]})["/type/domain/types"]]
    graph = _get_graph(s, types, True)
        
    return graph

def dump_type(s, type_id, follow_types=True):
    types = [type_id]
    graph = _get_graph(s, types, follow_types)
    
    return graph

def restore(s, graph, new_location, ignore_types=None):
    follow_types = graph.get("__follow_types", True)
    
    # create type dependencies
    type_requires_graph = {}
    
    # create prop dependencies
    prop_requires_graph = {}
    prop_to_type_map = {}
        
    for type_id, type_information in graph.iteritems():
        if not type_id.startswith("__"): # not a real type, but rather a helper
            # type dependency generation
            type_requires_graph[type_id] = type_information["__requires"]
            
            # prop dependency generation
            for prop in type_information["properties"]:
                prop_requires_graph[prop["id"]] = prop["__requires"]
                prop_to_type_map[prop["id"]] = type_id
            
    
    
    types_to_create = _generate_dependency_creation_order(type_requires_graph)
    props_to_create = _generate_dependency_creation_order(prop_requires_graph)
        
    origin_id, new_location_id = s.mqlreadmulti([{"id" : types_to_create[0], "type" : "/type/type", "domain" : {"id" : None}},
                                               {"id" : new_location, "a:id" : None}])                         
    origin_id = origin_id["domain"]["id"]
    new_location_id = new_location_id["a:id"]
    
    only_include = types_to_create + props_to_create
    
    for type_id in types_to_create:
        # let's find the type's key
        key = None
        for group in graph[type_id]["key"]:
            if group["namespace"] == origin_id:
                key = group["value"]
                break
        
        if key is None: # this shouldn't happen
            key = graph[type_id]["id"].split("/")[-1] # this can be wrong... different key than typeid
            
        
        tip = None
        if graph[type_id]["/freebase/documented_object/tip"]:
            tip = graph[type_id]["/freebase/documented_object/tip"]["value"]
        
        ignore = ("name", "domain", "key", "type", "id", "properties", "/freebase/type_hints/enumeration",
                    "/freebase/type_hints/included_types", "/freebase/type_hints/mediator", "/freebase/documented_object/tip")
        extra = _generate_extra_properties(graph[type_id], ignore)
        
        name = graph[type_id]["name"]["value"]
        included = [_convert_name_to_new(included_type["id"], origin_id, new_location_id, only_include) for included_type in graph[type_id]["/freebase/type_hints/included_types"]]
        cvt = graph[type_id]["/freebase/type_hints/mediator"]
        
        create_type(s, name, key, new_location_id, included=included, cvt=cvt, tip=tip, extra=extra)
    
    for prop_id in props_to_create: #* prop_id
        type_id = prop_to_type_map[prop_id]
        all_properties_for_type = graph[type_id]["properties"]
        for prop in all_properties_for_type:
            if prop["id"] == prop_id: # good, we are dealing with our specific property 
                
                new_schema = _convert_name_to_new(type_id, origin_id, new_location_id, only_include)
                
                name =  prop["name"]
                
                expected = None
                if prop["expected_type"]:
                    expected = _convert_name_to_new(prop["expected_type"], origin_id, new_location_id, only_include)
                
                for group in prop["key"]:
                    if group["namespace"] == type_id:
                        key = group["value"]
                        break
                
                tip = None
                if prop["/freebase/documented_object/tip"]:
                    tip = prop["/freebase/documented_object/tip"]["value"]
                
                disambig = prop["/freebase/property_hints/disambiguator"]
                unique = prop["unique"]
                
                ignore = ("name", "expected_type", "key", "id", "master_property", "delegated", "unique", "type", "schema",
                            "/freebase/property_hints/disambiguator", "enumeration", "/freebase/property_hints/enumeration", 
                            "/freebase/documented_object/tip")
                
                extra = _generate_extra_properties(prop, ignore)   
                
                if prop['master_property']:
                    converted_master_property = _convert_name_to_new(prop["master_property"], origin_id, new_location_id, only_include)
                    if converted_master_property == prop["master_property"]:
                        raise CVTError("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
                                        "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
                    reciprocate_property(s, name, key, converted_master_property,
                        unique, disambig=disambig, tip=tip, extra=extra)
                
                elif prop['delegated']:
                    delegate_property(s, _convert_name_to_new(prop['delegated'], origin_id, new_location_id, only_include), new_schema,
                        expected=expected, tip=tip, extra=extra)
                
                else:
                    create_property(s, name, key, new_schema, expected, unique, 
                        disambig=disambig, tip=tip, extra=extra) 
                        

def _get_graph(s, initial_types, follow_types):
    """ get the graph of dependencies of all the types involved, starting with a list supplied """
    
    assert isinstance(initial_types, (list, tuple))
    
    graph = {}
    to_update = set(initial_types)
    done = set()
    while len(to_update) > 0:
        new = to_update.pop()
        graph[new] = _get_needed(s, new)
        if follow_types:
            [to_update.add(b) for b in graph[new]["__related"] if b not in done]
            done.update(graph[new]["__related"])
        if not follow_types:
            # we have to check that there are no cvts attached to us, or else
            # ugly things happen (we can't include the cvt because the cvt won't link to us.)
            for prop in graph[new]["properties"]:
                if prop["master_property"]:
                    raise CVTError("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
                                    "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
    
    graph["__follow_types"] = follow_types
    return graph


def _convert_name_to_new(old_name, operating_base, new_base, only_include=None):
    if old_name in only_include and old_name.startswith(operating_base):
        return new_base + old_name.replace(operating_base, "", 1)
    else:
        return old_name

def _generate_dependency_creation_order(requires_graph):
    # This is a naive topographical sort to determine
    # in what order to create types or properties so
    # that the other type/properties they rely on
    # are already created
    
    # This function is called with the type dependencies
    # and then the property dependencies.
    
    
    # we sort the dependency_list because its a good idea
    # to create the guys with zero dependencies before the
    # guys with one.. it's just a simple optimization to 
    # the topographical sort
    dependency_list = [(len(requires), name) for (name, requires) in requires_graph.iteritems()]
    dependency_list.sort()
    
    creation_order_list = []
    while len(dependency_list) > 0:
        number_of_requirements, id = dependency_list.pop(0)
        if number_of_requirements == 0:
            creation_order_list.append(id)
            continue
        else:
            are_the_types_dependencies_already_resolved = True
            for requirement in requires_graph[id]:
                if requirement not in creation_order_list:
                    are_the_types_dependencies_already_resolved = False
                    continue
            if are_the_types_dependencies_already_resolved:
                creation_order_list.append(id)
            else:
                dependency_list.append((number_of_requirements, id))
    return creation_order_list

def _generate_extra_properties(dictionary_of_values, ignore):
    extra = {}
    for k, v in dictionary_of_values.iteritems():
        if k not in ignore and not k.startswith("__"):
            if v:
                if isinstance(v, basestring):
                    extra.update({k:v})
                elif isinstance(v, bool):
                    extra.update({k:v})
                elif v.has_key("id"):
                    extra.update({k:v["id"]})
                elif v.has_key("value"):
                    extra.update({k:v["value"]})
                else:
                    raise ValueError("There is a problem with getting the property value.")
    return extra

def _get_needed(s, type_id):
    q = TYPE_QUERY
    q.update(id=type_id)

    r = s.mqlread(q)
    properties = r.properties

    # let's identify who the parent is in order to only include
    # other types in that domain. We don't want to go around including
    # all of commons because someone's a /people/person
    parents = [r["domain"]["id"]]

    included_types = map(lambda x: x["id"], r["/freebase/type_hints/included_types"])
    related_types = set(included_types)
    for prop in properties:
        if prop["expected_type"]:
            related_types.add(prop["expected_type"])

    # we have two different types of relationships: required and related.
    # related can be used to generate subgraphs of types
    # required is used to generate the dependency graph of types

    related = _return_relevant(related_types, parents)
    requires = _return_relevant(included_types, parents)

    # get property information
    properties = r["properties"]
    for prop in properties:
        dependent_on = set()
        if prop["master_property"]:
            dependent_on.add(prop["master_property"])
        if prop["delegated"]:
            dependent_on.add(prop["delegated"])

        prop["__requires"] = _return_relevant(dependent_on, parents)

    # return all the information along with our special __* properties
    info = r
    info.update(__related=related, __requires=requires, __properties=properties)

    return info


def _return_relevant(start_list, parents):
    final = []
    for item in start_list:
        indomain = False
        for parent in parents:
            if item.startswith(parent):
                indomain = True
                continue
        if indomain:
            final.append(item)
    return final



PROPERTY_QUERY = {
        "optional" : True,
        "type" : "/type/property",
        "delegated" : None,
        "enumeration" : None, 
        "expected_type" : None,
        "id" : None,
        "key" : [{
            "namespace" : None, 
            "value" : None
        }],
        #"link" : [{}], 
        "master_property" : None,
        "name" : {"value" : None, "lang" : "/lang/en", "optional":True}, 
        "schema" : {"id" : None, "name" : None},
        "unique" : None, 
        "unit" : None,
        "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional" : True},
        "/freebase/property_hints/disambiguator" : None,
        "/freebase/property_hints/display_none" : None,
        "/freebase/property_hints/display_orientation" : None,
        "/freebase/property_hints/enumeration" : None,
        "/freebase/property_hints/dont_display_in_weblinks" : None,
        "/freebase/property_hints/inverse_description" : None,
    }

TYPE_QUERY = {
        "type" : "/type/type",
        "domain" : {},
        "key" : [{"namespace" : None, "value" : None}],
        "name" : {"value" : None, "lang" : "/lang/en", "optional":True},
        "/freebase/type_hints/included_types" : [{"id" : None, "optional" : True}],
        "/freebase/type_hints/mediator" : None,
        "/freebase/type_hints/enumeration" : None,
        "/freebase/type_hints/minor" : None,
        "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional":True},
        }
TYPE_QUERY.update(properties=[PROPERTY_QUERY])