path: root/freebase/schema.py

from copy import deepcopy
import sys
from freebase.api.session import HTTPMetawebSession, MetawebError
from freebase.api.session import LITERAL_TYPE_IDS

"""
NOTE
----
graph is used freely in this file. Some information:
 - It refers to an internal representation of a group of types.
 - It resembles a mqlread result, but it is not a mqlread result
 - It also has some helper variables like __requires and __related.
 - All helper variables start with __ since that's not valid MQL
 - It is produced by _get_graph
 - It can be converted into valid json (json.dumps(graph, indent=2))
 
Its structure is as follows:

    "type_id" : {
      "name" : "My Type",
      "id" : "/base_id/type_id"
      ...
      "__requires" : ["/base_id/type_id2"]
      "properties" : [
        {
          "id" : "/base_id/type_id/my_prop"
          ...
        },
        {
          ...
        }
      ]
    },
    "type_id2" : {...}
    ...
    
"""

class DelegationError(Exception):
    """You can't set the expected_type if the expected_type of the delegated (master) is a primitive"""

class CVTError(Exception):
    """You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true."""

def key_exists(s, k):
    q = {
        "id" : k,
        "guid" : None
    }
    return not None == s.mqlread(q)

### SCHEMA MANIPULATION ###
# Object helpers
def create_object(s, name="", path=None, key=None, namespace=None, 
                    included_types=None, create="unless_exists",
                    extra=None, use_permission_of=None, attribution=None):
    """
    Create object with name, a path or a key and namespace, included_types.
    You can also specify how it is created (unless_exists, unconditional..)
    as well as use_permission_of and attribution
    """
    if type(included_types) is str:
        included_types = [included_types]

    if path and (key or namespace):
        raise ValueError("You can't specify both the path and a key and namespace.")

    if path:
        key, namespace = get_key_namespace(path)

    if (key and not namespace) or (not key and namespace):
        raise ValueError("You must either specify both a key and a namespace, or neither.")

    if included_types:
        its = set(included_types)
        q = [{
            "id|=" : included_types,
            "/freebase/type_hints/included_types" : [{"id" : None}]
        }]
        for res in s.mqlread(q):
            its.update([x["id"] for x in res["/freebase/type_hints/included_types"]])

    wq = {
        "id" : None,
        "name" : name,
        "create" : create
    }

    # conditionally add key creation
    if key:
        wq.update({"key" : { 
            "namespace" : namespace,
            "value" : key,
        }})

    if included_types:
        wq.update(type = [{ "id" : it, "connect" : "insert" } for it in its])

    if extra: 
        wq.update(extra)

    return s.mqlwrite(wq, use_permission_of=use_permission_of, attribution_id=attribution)


def connect_object(s, id, newpath, extra=None, use_permission_of=None, attribution=None):
    """ connect object at id to a newpath.
    Example:
    connect_object(s, "/guid/002", "/en/the_beatles")
    """
    key, namespace = get_key_namespace(newpath)

    wq = {
        "id" : id,
        "key" : {
            "namespace" : namespace,
            "value" : key,
            "connect" : "insert"
        }
    }

    if extra: wq.update(extra)

    return s.mqlwrite(wq, use_permission_of=use_permission_of, attribution_id=attribution)


def disconnect_object(s, id, extra=None, use_permission_of=None, attribution=None):
    """ disconnect objects, as in removing their keys
    Example:
    disconnect_object(s, "/en/the_beatles")
    would remove the `the_beatles` key from /en """
    key, namespace = get_key_namespace(id)

    wq = {
        "id" : id,
        "key" : {
            "namespace" : namespace,
            "value" : key,
            "connect" : "delete"
        }
    }
    if extra: wq.update(extra)
    return s.mqlwrite(wq, use_permission_of=use_permission_of, attribution_id=attribution)

def move_object(s, oldpath, newpath, use_permission_of=None, attribution=None):
    """ move object from one key to another."""
    a = connect_object(s, oldpath, newpath, use_permission_of=use_permission_of, attribution=attribution)
    b = disconnect_object(s, oldpath, use_permission_of=use_permission_of, attribution=attribution)
    return a, b


def get_key_namespace(path):
    """ get (key, namespace) from a path
    get_key_namespace("/common") -> ("common", "/")
    get_key_namespace("/food/drinks") -> ("drinks", "/food")
    """
    # be careful with /common
    namespace, key = path.rsplit("/", 1)
    return (key, namespace or "/")


def add_type_to_object(s, id, type_id):
    """
     given an object (id) give it the type, type_id and all of its included types.
    """
    q = {
        "id" : type_id,
        "/freebase/type_hints/included_types" : [{"id" : None, "optional" : True}]
    }
    included_types = map(lambda x: x["id"], s.mqlread(q)["/freebase/type_hints/included_types"])
    
    wq = {
        "id" : id,
        "type" : [{
            "id" : it,
            "connect" : "insert"
        } for it in included_types + [type_id]]
    }
    return s.mqlwrite(wq)


def copy_property(s, id, newid, **extra):
    """ create a new property with the same information as the starting property """
    newname, newschema = get_key_namespace(newid)
    
    info = get_property_info(s, id)
    info["__raw"].update(extra)
    
    unique = None
    if info.has_key(unique):
        unique = info["unique"]
    disambig = None
    if info.has_key("/freebase/property_hints/disambiguator"):
        disambig = info["/freebase/property_hints/disambiguator"]
    create_property(s, info["name"], newname, newschema, info["expected_type"], unique=unique, disambig=disambig,
        tip=info["/freebase/documented_object/tip"], extra=info["__raw"])

def move_property(s, id, newid, **extra):
    """ create an identical property and delete the old one """
    copy_property(s, id, newid, **extra)
    disconnect_schema = {"type" : "/type/property", "schema" : {"connect" : "delete", "id" : "/".join(id.split("/")[:-1]) }}  
    disconnect_object(s, id, extra = disconnect_schema)

def get_property_info(s, prop_id):
    """
    get_property_info returns a valid json dictionary object that has all the information
    required to describe a property. This is only used by copy_property, but could be used
    by whoever.
    
    Ideally, all the required information by create_property is in the root of the dictionary
    while all the extra information is in result["__raw"]
    """
    q = deepcopy(PROPERTY_QUERY)
    q.update(id=prop_id)
    q.update(schema={"id" : None, "name" : None})
    res = s.mqlread(q)
    info = {}
    
    info["name"] = res["name"]["value"]
    if res["schema"]:
        info["schema"] = res["schema"]["id"]
    else: info["schema"] = None
    
    if res["key"]:
        info["key"] = [(x["value"], x["namespace"]) for x in res["key"]]
    else: info["key"] = None

    if res["/freebase/documented_object/tip"]:
        info["/freebase/documented_object/tip"] = res["/freebase/documented_object/tip"]["value"]
    else: info["/freebase/documented_object/tip"] = None
    
    ignore = ("optional", "type", "key", "/freebase/documented_object/tip")
    for prop in PROPERTY_QUERY.iterkeys():
        if prop not in ignore:
            if not info.has_key(prop):
                info[prop] = None

    info.update(_generate_extra_properties(res, ignore))
    
    # delete the properties that are going to be asked for in create_property
    del res["name"]
    del res["schema"]
    del res["key"]
    del res["expected_type"]
    del res["unique"]
    del res["/freebase/property_hints/disambiguator"]
    del res["/freebase/documented_object/tip"]
    
    # delete other useless things...
    del res["id"]

    for i in [k for k, v in res.iteritems() if v is None]:
        del res[i]

    info["__raw"] = res
    return info


# Create Type
def create_type(s, name, key, ns, cvt=False, tip=None, included=None, extra=None):
    """
    creates a type and takes care of associating it to its domain and attaching
    a key.
    """
    # TODO: CREATE SYNTHETIC VIEW
    if key_exists(s, ns + "/" + key ):
        return
    
    # assert isinstance(name, basestring) # name could be mqlish
    assert isinstance(key, basestring)
    assert isinstance(ns, basestring)
    
    assert tip is None or isinstance(tip, basestring)    
    assert included is None or isinstance(included, (basestring, list, tuple))
    assert extra is None or isinstance(extra, dict)

    wq = {
        "create" : "unconditional",
        "type" : "/type/type",
        "/type/type/domain" : { "connect" : "insert", "id" : ns },
        "name" : {"connect" : "insert", "value" : name, "lang" : "/lang/en" },
        "key" : {
            "connect" : "insert",
            "value" : key,
            "namespace" : ns
        }
    }

    if included:
        if isinstance(included, basestring):
            included = [included]
        its_q = [{
            "id|=" : included,
            "/freebase/type_hints/included_types" : [{"id" : None}]
        }]
        r = s.mqlread(its_q)
        included_types = set(included)
        if r:
            for i in r:
                included_types.update(map(lambda x: x["id"], i["/freebase/type_hints/included_types"]))

        its = [{"connect" : "insert", "id" : t} for t in included_types]
        wq['/freebase/type_hints/included_types'] = its

    # TODO: enum

    if cvt:
        wq['/freebase/type_hints/mediator'] = { "connect" : "update", "value" : True }
    if tip:
        wq['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" }
    
    if extra: wq.update(extra)
    return s.mqlwrite(wq, use_permission_of=ns)


# Create Property
def create_property(s, name, key, schema, expected, unique=False, disambig=False, tip=None, extra=None):
    """
    create a property with ect, unique, etc, and make schema and key links
    """
    if key_exists(s, schema + "/" + key):
        return

    # validate parameters
    # assert isinstance(name, basestring) # could be mql
    assert isinstance(key, basestring)
    assert isinstance(schema, basestring)
    assert isinstance(expected, basestring)
    assert tip is None or isinstance(tip, basestring)    
    assert extra is None or isinstance(extra, dict)
    
    wq = {
        "create" : "unconditional",
        "type" : "/type/property",
        "name" : name,
        "key" : {
            "connect" : "insert",
            "value" : key,
            "namespace" : { "id" : schema },
        },
        "schema" : { "connect" : "insert", "id" : schema },
        "expected_type" : { "connect" : "insert", "id" : expected }
    }
    if unique:
         wq['unique'] = { "connect" : "update", "value" : unique }
    if tip:
         wq['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" }
    if disambig:
         wq['/freebase/property_hints/disambiguator'] = { "connect" : "update", "value" : True }
    if extra:
         wq.update(extra)
    return s.mqlwrite(wq, use_permission_of=schema)

def delegate_property(s, p, schema, name=None, key=None, expected=None, tip=None, extra=None):
    """
    create a property with a delegate
    """
    assert isinstance(p, basestring)
    assert isinstance(schema, basestring)
    assert key is None or isinstance(key, basestring)
    assert expected is None or isinstance(expected, basestring)
    assert tip is None or isinstance(tip, basestring)
    
    q = {
        "id" : p,
        "type" : "/type/property",
        "name" : None,
        "unique" : None,
        "expected_type" : {"id" : None},
        "key" : None,
        "/freebase/documented_object/tip" : None,
        "/freebase/property_hints/disambiguator" : None
    }
    r = s.mqlread(q)

    # If the expected_type of the delegator(master) is a primitive, the delegated's
    # expected_type must be the same
    if r["expected_type"]["id"] in LITERAL_TYPE_IDS:
        if expected:
            if expected != r["expected_type"]["id"]:
                raise DelegationError("You can't set the expected_type if the expected_type of the delegated (master) is a primitive")
        expected = r["expected_type"]["id"]
    # If the expected_type of the delegator(master) is not a primitive, the delegated's
    # expected_type can be different
    elif expected is None:
        expected = r["expected_type"]["id"]


    if not tip and r["/freebase/documented_object/tip"]:
        tip = r["/freebase/documented_object/tip"]

    if name is None:
        name = r["name"]
    if key is None:
        key = r["key"]
    
    delegate = { "/type/property/delegated" : p}
    if extra: delegate.update(extra)
    
    return create_property(s, name, key, schema, expected, r['unique'],
        r["/freebase/property_hints/disambiguator"],
        tip,
        delegate)

def reciprocate_property(s, name, key, master, unique=False, disambig=False, tip=None, extra=None):
    """ We're creating a reciprocate property of the master property. Let's illustrate
    the idea behind the function with an example.

    Say we examine the /visual_art/art_period_movement/associated_artworks property.
    An example of an art_period_movement is the Renaissance, and once associated_artworks
    could be the /en/mona_lisa. In this example, /visual_art/art_period_movement/associated_artworks
    will be the master property, and /visual_art/artwork/period_or_movement will be the reciprocal.

    In order to determine the characterists of the reciprocal property, we must examine the master.
    associated_artworks property's schema is /visual_art/art_period_movement and its expected
    type is /visual_art/artwork. Notice the similarity to /visual_art/artwork/period_or_movement.
    period_or_movement's schema is /visual_art/artwork -- art_period_movement's expected type.
    period_or_movement's expected type is /visual_art/art_period_movement -- art_period_movement's
    schema!

    So, given a master, the reciprocal's schema is the master's expected type and the reciprocal's
    expected type is the master's schema. """

    # assert isinstance(name, basestring) # name could be mqlish
    assert isinstance(key, basestring)
    assert isinstance(master, basestring)
    
    assert tip is None or isinstance(tip, basestring)
    assert extra is None or isinstance(extra, dict)
    
    # get master information
    q = {
        "id" : master,
        "/type/property/expected_type" : None,
        "/type/property/schema" : None,
        "/type/property/reverse_property" : None }
    r = s.mqlread(q)
    ect = r["/type/property/expected_type"]
    schema = r["/type/property/schema"]

    # check to see if a master existed
    if r["/type/property/reverse_property"]:
        raise MetawebError("You can't reciprocate property %s who \
                            already has a reverse property %s",
                            (master, r["/type/property/reverse_property"]))


    master = {"master_property" : master}
    if extra: master.update(extra)

    # NOTE: swapping ect and schema; see comment above
    return create_property(s, name, key, ect, schema, unique, disambig, tip,
        extra = master)

# dump / restore types
def dump_base(s, base_id):
    """ dump a base into a `graph` object. See information at the top of the file for more
    information on the graph file """
    domain_types = s.mqlread({"id" : base_id,
                              "/type/domain/types": [{"id" : None}]})
    types = [type_object["id"]
             for type_object in domain_types["/type/domain/types"]]
    graph = _get_graph(s, types, True)
        
    return graph

def dump_type(s, type_id, follow_types=True):
    """ dump a type (similar to dump_base) and has an argument follow_types that determines
    if it should dump types that are neccessary or just rely on them """
    types = [type_id]
    graph = _get_graph(s, types, follow_types)
    
    return graph

def restore(s, graph, new_location, ignore_types=None):
    """ given a `graph` object and a another location, we can upload our graph output
    into a new location in the freebase graph """
    follow_types = graph.get("__follow_types", True)
    
    # We assume the new_location is empty. if it isn't, we bail.
    # well, ...
    
    
    # create type dependencies
    type_requires_graph = {}
    
    # create prop dependencies
    prop_requires_graph = {}
    prop_to_type_map = {}
        
    for type_id, type_information in graph.iteritems():
        if not type_id.startswith("__"): # not a real type, but rather a helper
            # type dependency generation
            type_requires_graph[type_id] = type_information["__requires"]
            
            # prop dependency generation
            for prop in type_information["properties"]:
                prop_requires_graph[prop["id"]] = prop["__requires"]
                prop_to_type_map[prop["id"]] = type_id
            
    
    
    types_to_create = _generate_dependency_creation_order(type_requires_graph)
    props_to_create = _generate_dependency_creation_order(prop_requires_graph)

    # make sure we're starting fresh - sometimes your mwLastWriteTime
    # isn't fresh if create_private_domain fails
    s.touch()
    origin_id, new_location_id = s.mqlreadmulti([{"id" : types_to_create[0],
                                                  "type" : "/type/type",
                                                  "domain" : {"id" : None}},
                                                 {"id" : new_location,
                                                  "a:id" : None}])

    if new_location_id is None:
        # create the domain if it doesnt' exist already
        username = s.user_info()["username"]
        user_id = "/user/%s" % username
        if not new_location.startswith("%s/" % user_id):
            sys.stderr.write("%s does not exist: If creating a domain outside of %s, you must create it yourself\n" % (new_location, user_id))
            sys.exit(1)

        location_key = new_location[len(user_id)+1:]
        tail_key = new_location.rsplit("/", 1)[-1]
        if tail_key != location_key:
            sys.stderr.write("%s does not exist: can only create domains as direct children of %s\n" % (new_location, user_id))
            sys.exit(1)
        s.create_private_domain(location_key, location_key)
    
    origin_id = origin_id["domain"]["id"]
    new_location_id = new_location_id["a:id"]
    
    only_include = types_to_create + props_to_create
    
    for type_id in types_to_create:
        # let's find the type's key
        key = None
        for group in graph[type_id]["key"]:
            if group["namespace"] == origin_id:
                key = group["value"]
                break
        
        if key is None: # this shouldn't happen
            key = graph[type_id]["id"].split("/")[-1] # this can be wrong... different key than typeid
            
        
        tip = None
        if graph[type_id]["/freebase/documented_object/tip"]:
            tip = graph[type_id]["/freebase/documented_object/tip"]["value"]
        
        ignore = TYPE_INGORE_PROPERTIES
        extra = _generate_extra_properties(graph[type_id], ignore)
        
        name = graph[type_id]["name"]["value"]
        included = [_convert_name_to_new(included_type["id"], origin_id, new_location_id, only_include)
                    for included_type in graph[type_id]["/freebase/type_hints/included_types"]]
        cvt = graph[type_id]["/freebase/type_hints/mediator"]
        
        create_type(s, name, key, new_location_id, included=included, cvt=cvt, tip=tip, extra=extra)
    
    for prop_id in props_to_create:
        type_id = prop_to_type_map[prop_id]
        all_properties_for_type = graph[type_id]["properties"]
        for prop in all_properties_for_type:
            if prop["id"] == prop_id: # good, we are dealing with our specific property 
                
                new_schema = _convert_name_to_new(type_id, origin_id, new_location_id, only_include)
                
                name =  prop["name"]
                
                expected = None
                if prop["expected_type"]:
                    expected = _convert_name_to_new(prop["expected_type"], origin_id, new_location_id, only_include)
                
                for group in prop["key"]:
                    if group["namespace"] == type_id:
                        key = group["value"]
                        break
                
                tip = None
                if prop["/freebase/documented_object/tip"]:
                    tip = prop["/freebase/documented_object/tip"]["value"]
                
                disambig = prop["/freebase/property_hints/disambiguator"]
                unique = prop["unique"]
                
                ignore = PROPERTY_IGNORE_PROPERTIES
                extra = _generate_extra_properties(prop, ignore)   
                
                if prop['master_property']:
                    converted_master_property = _convert_name_to_new(prop["master_property"], origin_id, new_location_id, only_include)
                    if converted_master_property == prop["master_property"]:
                        raise CVTError("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
                                       "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
                    reciprocate_property(s, name, key, converted_master_property,
                        unique, disambig=disambig, tip=tip, extra=extra)
                
                elif prop['delegated']:
                    delegate_property(s, _convert_name_to_new(prop['delegated'], origin_id, new_location_id, only_include), new_schema,
                        expected=expected, tip=tip, extra=extra)
                
                else:
                    create_property(s, name, key, new_schema, expected, unique,
                        disambig=disambig, tip=tip, extra=extra)

def _get_graph(s, initial_types, follow_types):
    """ get the graph of dependencies of all the types involved, starting with a list supplied """
    
    assert isinstance(initial_types, (list, tuple))
    
    graph = {}
    to_update = set(initial_types)
    done = set()
    while len(to_update) > 0:
        new = to_update.pop()
        graph[new] = _get_needed(s, new)
        if follow_types:
            [to_update.add(b) for b in graph[new]["__related"] if b not in done]
            done.update(graph[new]["__related"])
        if not follow_types:
            # we have to check that there are no cvts attached to us, or else
            # ugly things happen (we can't include the cvt because the cvt won't link to us.)
            for prop in graph[new]["properties"]:
                if prop["master_property"]:
                    raise CVTError("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \
                                    "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"]))
    
    graph["__follow_types"] = follow_types
    return graph


def _convert_name_to_new(old_name, operating_base, new_base, only_include=None):
    if old_name in only_include and old_name.startswith(operating_base):
        return new_base + old_name.replace(operating_base, "", 1)
    else:
        return old_name

def _generate_dependency_creation_order(requires_graph):
    # This is a naive topographical sort to determine
    # in what order to create types or properties so
    # that the other type/properties they rely on
    # are already created
    
    # This function is called with the type dependencies
    # and then the property dependencies.
    
    
    # we sort the dependency_list because its a good idea
    # to create the guys with zero dependencies before the
    # guys with one.. it's just a simple optimization to 
    # the topographical sort
    dependency_list = [(len(requires), name) for (name, requires) in requires_graph.iteritems()]
    dependency_list.sort()
    
    creation_order_list = []
    while len(dependency_list) > 0:
        number_of_requirements, id = dependency_list.pop(0)
        if number_of_requirements == 0:
            creation_order_list.append(id)
            continue
        else:
            are_the_types_dependencies_already_resolved = True
            for requirement in requires_graph[id]:
                if requirement not in creation_order_list:
                    are_the_types_dependencies_already_resolved = False
                    continue
            if are_the_types_dependencies_already_resolved:
                creation_order_list.append(id)
            else:
                dependency_list.append((number_of_requirements, id))
    return creation_order_list

def _generate_extra_properties(dictionary_of_values, ignore):
    extra = {}
    for k, v in dictionary_of_values.iteritems():
        if k not in ignore and not k.startswith("__"):
            if v:
                if isinstance(v, basestring):
                    extra.update({k:v})
                elif isinstance(v, bool):
                    extra.update({k:v})
                elif v.has_key("id"):
                    extra.update({k:v["id"]})
                elif v.has_key("value"):
                    extra.update({k:v["value"]})
                else:
                    raise ValueError("There is a problem with getting the property value.")
            else:
                if isinstance(v, bool): # well, if its False...
                    extra.update({k:v})
    return extra

def _get_needed(s, type_id):
    q = deepcopy(TYPE_QUERY)
    q.update(id=type_id)

    r = s.mqlread(q)
    properties = r.properties

    # let's identify who the parent is in order to only include
    # other types in that domain. We don't want to go around including
    # all of commons because someone's a /people/person
    parents = [r["domain"]["id"]]

    included_types = map(lambda x: x["id"], r["/freebase/type_hints/included_types"])
    related_types = set(included_types)
    for prop in properties:
        if prop["expected_type"]:
            related_types.add(prop["expected_type"])

    # we have two different types of relationships: required and related.
    # related can be used to generate subgraphs of types
    # required is used to generate the dependency graph of types

    related = _return_relevant(related_types, parents)
    requires = _return_relevant(included_types, parents)

    # get property information
    properties = r["properties"]
    for prop in properties:
        dependent_on = set()
        if prop["master_property"]:
            dependent_on.add(prop["master_property"])
        if prop["delegated"]:
            dependent_on.add(prop["delegated"])

        prop["__requires"] = _return_relevant(dependent_on, parents)

    # return all the information along with our special __* properties
    info = r
    info.update(__related=related, __requires=requires)

    return info


def _return_relevant(start_list, parents):
    final = []
    for item in start_list:
        indomain = False
        for parent in parents:
            if item.startswith(parent):
                indomain = True
                continue
        if indomain:
            final.append(item)
    return final

PROPERTY_QUERY = {
        "optional" : True,
        "type" : "/type/property",
        "delegated" : None,
        "enumeration" : None, 
        "expected_type" : None,
        "id" : None,
        "key" : [{
            "namespace" : None, 
            "value" : None
        }],
        "master_property" : None,
        "name" : {"value" : None, "lang" : "/lang/en", "optional":True}, 
        "unique" : None, 
        "unit" : None,
        "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional" : True},
        "/freebase/property_hints/disambiguator" : None,
        "/freebase/property_hints/display_none" : None,
        "/freebase/property_hints/display_orientation" : None,
        "/freebase/property_hints/enumeration" : None,
        "/freebase/property_hints/dont_display_in_weblinks" : None,
        "/freebase/property_hints/inverse_description" : None,
    }

TYPE_QUERY = {
        "type" : "/type/type",
        "domain" : {},
        "key" : [{"namespace" : None, "value" : None}],
        "name" : {"value" : None, "lang" : "/lang/en", "optional":True},
        "/freebase/type_hints/included_types" : [{"id" : None, "optional" : True}],
        "/freebase/type_hints/mediator" : None,
        "/freebase/type_hints/enumeration" : None,
        "/freebase/type_hints/minor" : None,
        "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional":True},
        }
TYPE_QUERY.update(properties=[deepcopy(PROPERTY_QUERY)])    

TYPE_INGORE_PROPERTIES = ("name", "domain", "key", "type", "id", "properties", "/freebase/type_hints/enumeration",
            "/freebase/type_hints/included_types", "/freebase/type_hints/mediator", "/freebase/documented_object/tip")

PROPERTY_IGNORE_PROPERTIES = ("name", "expected_type", "key", "id", "master_property", "delegated", "unique", "type", "schema",
            "/freebase/property_hints/disambiguator", "enumeration", "/freebase/property_hints/enumeration", 
            "/freebase/documented_object/tip")