from freebase.api.session import HTTPMetawebSession from freebase.api.session import get_key_namespace, LITERAL_TYPE_IDS def key_exists(s, k): q = { "id" : k, "guid" : None } return not None == s.mqlread(q) def type_object(s, id, type_id): q = { "id" : type_id, "/freebase/type_hints/included_types" : [{"id" : None}] } included_types = map(lambda x: x["id"], s.mqlread(q)["/freebase/type_hints/included_types"]) wq = { "id" : id, "type" : [{ "id" : it, "connect" : "insert" } for it in included_types + [type_id]] } return s.mqlwrite(wq) def copy_property(s, id, newid, **extra): newname, newschema = get_key_namespace(newid) info = get_property_info(s, id) info["__raw"].update(extra) create_property(s, info["name"], newname, newschema, info["expected_type"], info["unique"], info["/freebase/property_hints/disambiguator"], info["/freebase/documented_object/tip"], info["__raw"]) def move_property(s, id, newid, **extra): copy_property(s, id, newid, **extra) disconnect_schema = {"type" : "/type/property", "schema" : {"connect" : "delete", "id" : "/".join(id.split("/")[:-1]) }} s.disconnect_object(id, extra = disconnect_schema) def get_property_info(s, prop_id): q = PROPERTY_QUERY q.update(id=prop_id) res = s.mqlread(q) info = {} info["name"] = res["name"]["value"] if res["schema"]: info["schema"] = res["schema"]["id"] else: info["schema"] = None if res["key"]: info["keys"] = map(lambda x: (x["value"], x["namespace"]), res["key"]) else: info["key"] = None if res["/freebase/documented_object/tip"]: info["/freebase/documented_object/tip"] = res["/freebase/documented_object/tip"]["value"] else: info["/freebase/documented_object/tip"] = None for i in ["delegated", "enumeration", "expected_type", "id", "master_property", "unique", "unit", "/freebase/property_hints/disambiguator", "/freebase/property_hints/display_none", "/freebase/property_hints/display_orientation","/freebase/property_hints/enumeration", "/freebase/property_hints/dont_display_in_weblinks", "/freebase/property_hints/inverse_description"]: if res[i]: if isinstance(res[i], basestring): info[i] = res[i] elif isinstance(res[i], bool): info[i] = res[i] elif res[i].has_key("id"): info[i] = res[i]["id"] elif res[i].has_key("value"): info[i] = res[i]["value"] else: raise Exception("There is a problem with getting the property value.") else: info[i] = None # delete the properties that are going to be asked for in create_property del res["name"] del res["schema"] del res["key"] del res["expected_type"] del res["unique"] del res["/freebase/property_hints/disambiguator"] del res["/freebase/documented_object/tip"] # delete other useless things... del res["id"] for i in [k for k, v in res.items() if v is None]: del res[i] info["__raw"] = res return info # Create Type def create_type(s, name, key, ns, cvt=None, tip=None, included=None, extra=None): if key_exists(s, ns + "/" + key ): return q = { "create" : "unconditional", "type" : "/type/type", "/type/type/domain" : { "connect" : "insert", "id" : ns }, "name" : {"connect" : "insert", "value" : name, "lang" : "/lang/en" }, "key" : { "connect" : "insert", "value" : key, "namespace" : ns } } if included: if isinstance(included, basestring): included = [included] itsq = [{ "id|=" : included, "/freebase/type_hints/included_types" : [{"id" : None}] }] r = s.mqlread(itsq) included_types = set(included) if r: for i in r: included_types.update(map(lambda x: x["id"], i["/freebase/type_hints/included_types"])) its = [{"connect" : "insert", "id" : t} for t in included_types] q['/freebase/type_hints/included_types'] = its # TODO: enum if cvt: q['/freebase/type_hints/mediator'] = { "connect" : "update", "value" : True } if tip: q['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" } if extra: q.update(extra) return s.mqlwrite(q, use_permission_of=ns) # Create Property def create_property(s, name, key, schema, expected, unique=False, disambig=False, tip=None, extra=None): if key_exists(s, schema + "/" + key): # this isn't (neccesarily) a problem # raise Exception("The key \"%s\" already exists!" % (schema + "/" + key)) return # validate parameters str q = { "create" : "unconditional", "type" : "/type/property", "name" : name, "key" : { "connect" : "insert", "value" : key, "namespace" : { "id" : schema }, }, "schema" : { "connect" : "insert", "id" : schema }, "expected_type" : { "connect" : "insert", "id" : expected } } if unique: q['unique'] = { "connect" : "update", "value" : unique } if tip: q['/freebase/documented_object/tip'] = { "connect" : "update", "value" : tip, "lang" : "/lang/en" } if disambig: q['/freebase/property_hints/disambiguator'] = { "connect" : "update", "value" : True } if extra: q.update(extra) #print json.dumps(q, indent=2) return s.mqlwrite(q, use_permission_of=schema) def delegate_property(s, p, schema, name=None, key=None, expected=None, tip=None, extra=None): q = { "id" : p, "type" : "/type/property", "name" : None, "unique" : None, "expected_type" : {"id" : None}, "key" : None, "/freebase/documented_object/tip" : None, "/freebase/property_hints/disambiguator" : None } r = s.mqlread(q) # If the expected_type of the delegator(master) is a primitive, the delegated's # expected_type must be the same if r["expected_type"]["id"] in LITERAL_TYPE_IDS: if expected: if expected != r["expected_type"]["id"]: raise Exception("You can't set the expected_type if the expected_type of the delegated (master) is a primitive") expected = r["expected_type"]["id"] # If the expected_type of the delegator(master) is not a primitive, the delegated's # expected_type can be different elif expected is None: expected = r["expected_type"]["id"] if not tip and r["/freebase/documented_object/tip"]: tip = r["/freebase/documented_object/tip"] if name is None: name = r["name"] if key is None: key = r["key"] delegate = { "/type/property/delegated" : p} if extra: delegate.update(extra) return create_property(s, name, key, schema, expected, r['unique'], r["/freebase/property_hints/disambiguator"], tip, delegate) def reciprocate_property(s, name, key, master, unique=False, disambig=False, tip=None, extra=None): """ We're creating a reciprocate property of the master property. Let's illustrate the idea behind the function with an example. Say we examine the /visual_art/art_period_movement/associated_artworks property. An example of an art_period_movement is the Renaissance, and once associated_artworks could be the /en/mona_lisa. In this example, /visual_art/art_period_movement/associated_artworks will be the master property, and /visual_art/artwork/period_or_movement will be the reciprocal. In order to determine the characterists of the reciprocal property, we must examine the master. associated_artworks property's schema is /visual_art/art_period_movement and its expected type is /visual_art/artwork. Notice the similarity to /visual_art/artwork/period_or_movement. period_or_movement's schema is /visual_art/artwork -- art_period_movement's expected type. period_or_movement's expected type is /visual_art/art_period_movement -- art_period_movement's schema! So, given a master, the reciprocal's schema is the master's expected type and the reciprocal's expected type is the master's schema. """ q = { "id" : master, "/type/property/expected_type" : None, "/type/property/schema" : None } r = s.mqlread(q) ect = r["/type/property/expected_type"] schema = r["/type/property/schema"] master = {"master_property" : master} if extra: master.update(extra) # NOTE: swapping ect and schema; see comment above return create_property(s, name, key, ect, schema, unique, disambig, tip, extra = master) # upload / restore types def dump_base(s, base_id): types = map(lambda x: x["id"], s.mqlread({"id" : base_id, "/type/domain/types":[{"id" : None}]})["/type/domain/types"]) graph = _get_graph(s, types, True) graph["__follow_types"] = True return graph def dump_type(s, type_id, follow_types=True): types = [type_id] graph = _get_graph(s, types, follow_types) graph["__follow_types"] = follow_types return graph def restore(s, graph, new_location, ignore_types=None, debug=False): follow_types = graph.get("__follow_types", True) if debug: print "Following types:", follow_types # create type dependencies typegraph = {} for tid, idres in graph.items(): if not tid.startswith("__"): typegraph[tid] = idres["__requires"] type_deps = map(lambda (name, x): (len(x), name), typegraph.iteritems()) type_deps.sort() if follow_types: types_to_create = create_what(type_deps, typegraph) else: types_to_create = typegraph.keys() # create property dependencies propgraph = {} proptotype = {} for tid, idres in graph.items(): if not tid.startswith("__"): for prop in idres["properties"]: propgraph[prop["id"]] = prop["__requires"] proptotype[prop["id"]] = tid prop_deps = map(lambda (name, x): (len(x), name), propgraph.items()) #* prop_deps.sort() if follow_types: props_to_create = create_what(prop_deps, propgraph) else: props_to_create = propgraph.keys() if debug: print "types", types_to_create if debug: print "-----------------------" if debug: print "props", props_to_create base_id, domain_id = s.mqlreadmulti([{"id" : types_to_create[0], "type" : "/type/type", "domain" : {"id" : None}}, {"id" : new_location, "a:id" : None}]) base_id = base_id["domain"]["id"] domain_id = domain_id["a:id"] only_include = types_to_create + props_to_create for type in types_to_create: if debug: print type key = "" if len(graph[type]["key"]) == 1: key = graph[type]["key"][0]["value"] else: expectedname = graph[type]["id"].split("/")[-1] if base_id: for group in graph[type]["key"]: if group["namespace"] == base_id: key = group["value"] continue if key is None: key = expectedname tip = None if graph[type]["/freebase/documented_object/tip"]: tip = graph[type]["/freebase/documented_object/tip"]["value"] ignore = ("name", "domain", "key", "type", "id", "properties", "/freebase/type_hints/enumeration", "/freebase/type_hints/included_types", "/freebase/type_hints/mediator", "/freebase/documented_object/tip") extra = {} for k, v in graph[type].items(): if k not in ignore and not k.startswith("__"): if v: if isinstance(v, basestring): extra.update({k:v}) elif isinstance(v, bool): extra.update({k:v}) elif v.has_key("id"): extra.update({k:v["id"]}) elif v.has_key("value"): extra.update({k:v["value"]}) else: raise Exception("There is a problem with getting the property value.") create_type(s, graph[type]["name"]["value"], key, domain_id, included=map(lambda x: convert_name(x["id"], base_id, domain_id, only_include), graph[type]["/freebase/type_hints/included_types"]), cvt=graph[type]["/freebase/type_hints/mediator"], tip=tip, extra=extra) if debug: print "--------------------------" for prop in props_to_create: #* prop_id info = graph[proptotype[prop]]["properties"] for i in info: if i["id"] == prop: schema = convert_name(proptotype[prop], base_id, domain_id, only_include) if debug: print prop expected = None if i["expected_type"]: expected = convert_name(i["expected_type"], base_id, domain_id, only_include) for k in i["key"]: if k["namespace"] == proptotype[prop]: key = k["value"] if i["/freebase/documented_object/tip"]: tip = graph[type]["/freebase/documented_object/tip"] disambig = i["/freebase/property_hints/disambiguator"] ignore = ("name", "expected_type", "key", "id", "master_property", "delegated", "unique", "type", "schema", "/freebase/property_hints/disambiguator", "enumeration", "/freebase/property_hints/enumeration", "/freebase/documented_object/tip") extra = {} for k, v in i.items(): if k not in ignore and not k.startswith("__"): if v: if isinstance(v, basestring): extra.update({k:v}) elif isinstance(v, bool): extra.update({k:v}) elif v.has_key("id"): extra.update({k:v["id"]}) elif v.has_key("value"): extra.update({k:v["value"]}) else: raise Exception("There is a problem with getting the property value.") # since we are creating a property, all these connect insert delicacies are unneccesary """if isinstance(v, basestring) and v.startswith("/"): # an id extra.update({k : {"connect" : "insert", "id" : v}}) elif isinstance(v, bool): # a bool value extra.update({k : {"connect" : "insert", "value" : v}}) else: # an english value extra.update({k : {"connect" : "insert", "value" : v, "lang" : "/lang/en"}})""" if i['master_property']: converted_master_property = convert_name(i["master_property"], base_id, domain_id, only_include) if converted_master_property == i["master_property"]: raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \ "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"])) reciprocate_property(s, i["name"], key, converted_master_property, i["unique"], disambig=disambig, tip=tip, extra=extra) elif i['delegated']: delegate_property(s, convert_name(i['delegated'], base_id, domain_id, only_include), schema, expected=expected, tip=tip, extra=extra) else: create_property(s, i["name"], key, schema, expected, i["unique"], disambig=disambig, tip=tip, extra=extra) def _get_graph(s, initial_types, follow_types): """ get the graph of dependencies of all the types involved, starting with a list supplied """ assert isinstance(initial_types, (list, tuple)) graph = {} to_update = set(initial_types) done = set() while len(to_update) > 0: new = to_update.pop() graph[new] = _get_needed(s, new) if follow_types: [to_update.add(b) for b in graph[new]["__related"] if b not in done] done.update(graph[new]["__related"]) if not follow_types: # we have to check that there are no cvts attached to us, or else # ugly things happen (we can't include the cvt) for prop in graph[new]["properties"]: if prop["master_property"]: raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \ "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"])) return graph def convert_name(old_name, operating_base, new_base, only_include=None): if old_name in only_include and old_name.startswith(operating_base): return new_base + old_name.replace(operating_base, "", 1) else: return old_name def create_what(deps, graph): create_list = [] while len(deps) > 0: neediness, id = deps.pop(0) if neediness == 0: create_list.append(id) continue else: work = True for req in graph[id]: if req not in create_list: work = False continue if work: create_list.append(id) else: deps.append((neediness, id)) return create_list def _get_needed(s, type_id): q = TYPE_QUERY q.update(id=type_id) r = s.mqlread(q) properties = r.properties # let's identify who the parent is in order to only include # other types in that domain. We don't want to go around including # all of commons because someone's a /people/person parents = [r["domain"]["id"]] included_types = map(lambda x: x["id"], r["/freebase/type_hints/included_types"]) related_types = set(included_types) for prop in properties: if prop["expected_type"]: related_types.add(prop["expected_type"]) # we have two different types of relationships: required and related. # related can be used to generate subgraphs of types # required is used to generate the dependency graph of types related = return_relevant(related_types, parents) requires = return_relevant(included_types, parents) # get property information properties = r["properties"] for prop in properties: dependent_on = set() if prop["master_property"]: dependent_on.add(prop["master_property"]) if prop["delegated"]: dependent_on.add(prop["delegated"]) prop["__requires"] = return_relevant(dependent_on, parents) # return all the information along with our special __* properties info = r info.update(__related=related, __requires=requires, __properties=properties) return info def return_relevant(start_list, parents): final = [] for item in start_list: indomain = False for parent in parents: if item.startswith(parent): indomain = True continue if indomain: final.append(item) return final PROPERTY_QUERY = { "optional" : True, "type" : "/type/property", "delegated" : None, "enumeration" : None, "expected_type" : None, "id" : None, "key" : [{ "namespace" : None, "value" : None }], #"link" : [{}], "master_property" : None, "name" : {"value" : None, "lang" : "/lang/en", "optional":True}, "schema" : {"id" : None, "name" : None}, "unique" : None, "unit" : None, "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional" : True}, "/freebase/property_hints/disambiguator" : None, "/freebase/property_hints/display_none" : None, "/freebase/property_hints/display_orientation" : None, "/freebase/property_hints/enumeration" : None, "/freebase/property_hints/dont_display_in_weblinks" : None, "/freebase/property_hints/inverse_description" : None, } TYPE_QUERY = { "type" : "/type/type", "domain" : {}, "key" : [{"namespace" : None, "value" : None}], "name" : {"value" : None, "lang" : "/lang/en", "optional":True}, "/freebase/type_hints/included_types" : [{"id" : None, "optional" : True}], "/freebase/type_hints/mediator" : None, "/freebase/type_hints/enumeration" : None, "/freebase/type_hints/minor" : None, "/freebase/documented_object/tip" : {"value" : None, "limit":1, "optional":True}, } TYPE_QUERY.update(properties=[PROPERTY_QUERY])