# TODO: copy /freebase/type_hints junk # TODO: preserve property order. from pprint import pprint import random import json from freebase.schema import TYPE_QUERY, PROPERTY_QUERY from freebase.api import HTTPMetawebSession, MetawebError from freebase.schema import create_type, create_property, reciprocate_property, delegate_property s = HTTPMetawebSession("http://sandbox-freebase.com") s.login("nitromaster101@gmail.com", "something") import time def dump_base(s, base_id): types = map(lambda x: x["id"], s.mqlread({"id" : base_id, "/type/domain/types":[{"id" : None}]})["/type/domain/types"]) graph = _get_graph(types) graph["__follow_types"] = True return graph def dump_type(s, type_id, follow_types=True): types = [type_id] graph = _get_graph(types, follow_types) graph["__follow_types"] = follow_types result = json.dumps(graph, indent=2) fh = open("junk.json", "w") fh.write(result) fh.close() return graph def upload_type(s, graph, new_location, ignore_types=None, debug=False): follow_types = graph.get("__follow_types", True) if debug: print "Following types:", follow_types # create type dependencies typegraph = {} for tid, idres in graph.items(): if not tid.startswith("__"): typegraph[tid] = idres["__requires"] type_deps = map(lambda (name, x): (len(x), name), typegraph.items()) type_deps.sort() if follow_types: types_to_create = create_what(type_deps, typegraph) else: types_to_create = typegraph.keys() # create property dependencies propgraph = {} proptotype = {} for tid, idres in graph.items(): if not tid.startswith("__"): for prop in idres["properties"]: propgraph[prop["id"]] = prop["__requires"] proptotype[prop["id"]] = tid prop_deps = map(lambda (name, x): (len(x), name), propgraph.items()) prop_deps.sort() if follow_types: props_to_create = create_what(prop_deps, propgraph) else: props_to_create = propgraph.keys() if debug: print "types", types_to_create if debug: print "-----------------------" if debug: print "props", props_to_create base_id, domain_id = s.mqlreadmulti([{"id" : types_to_create[0], "type" : "/type/type", "domain" : {"id" : None}}, {"id" : new_location, "a:id" : None}]) base_id = base_id["domain"]["id"] domain_id = domain_id["a:id"] only_include = types_to_create + props_to_create for type in types_to_create: if debug: print type key = "" if len(graph[type]["key"]) == 1: key = graph[type]["key"][0]["value"] else: expectedname = graph[type]["id"].split("/")[-1] if base_id: for group in graph[type]["key"]: if group["namespace"] == base_id: key = group["value"] continue if key is None: key = expectedname tip = None if graph[type]["/freebase/documented_object/tip"]: tip = graph[type]["/freebase/documented_object/tip"]["value"] ignore = ("name", "domain", "key", "type", "id", "properties", "/freebase/type_hints/enumeration", "/freebase/type_hints/included_types", "/freebase/type_hints/mediator", "/freebase/documented_object/tip") extra = {} for k, v in graph[type].items(): if k not in ignore and not k.startswith("__"): if v: if isinstance(v, basestring): extra.update({k:v}) elif isinstance(v, bool): extra.update({k:v}) elif v.has_key("id"): extra.update({k:v["id"]}) elif v.has_key("value"): extra.update({k:v["value"]}) else: raise Exception("There is a problem with getting the property value.") create_type(s, graph[type]["name"]["value"], key, domain_id, included=map(lambda x: convert_name(x["id"], base_id, domain_id, only_include), graph[type]["/freebase/type_hints/included_types"]), cvt=graph[type]["/freebase/type_hints/mediator"], tip=tip, extra=extra) if debug: print "--------------------------" for prop in props_to_create: info = graph[proptotype[prop]]["properties"] for i in info: if i["id"] == prop: schema = convert_name(proptotype[prop], base_id, domain_id, only_include) if debug: print prop expected = None if i["expected_type"]: expected = convert_name(i["expected_type"], base_id, domain_id, only_include) for k in i["key"]: if k.namespace == proptotype[prop]: key = k.value if i["/freebase/documented_object/tip"]: tip = graph[type]["/freebase/documented_object/tip"] disambig = i["/freebase/property_hints/disambiguator"] ignore = ("name", "expected_type", "key", "id", "master_property", "delegated", "unique", "type", "schema", "/freebase/property_hints/disambiguator", "enumeration", "/freebase/property_hints/enumeration", "/freebase/documented_object/tip") extra = {} for k, v in i.items(): if k not in ignore and not k.startswith("__"): if v: if isinstance(v, basestring): extra.update({k:v}) elif isinstance(v, bool): extra.update({k:v}) elif v.has_key("id"): extra.update({k:v["id"]}) elif v.has_key("value"): extra.update({k:v["value"]}) else: raise Exception("There is a problem with getting the property value.") # since we are creating a property, all these connect insert delicacies are unneccesary """if isinstance(v, basestring) and v.startswith("/"): # an id extra.update({k : {"connect" : "insert", "id" : v}}) elif isinstance(v, bool): # a bool value extra.update({k : {"connect" : "insert", "value" : v}}) else: # an english value extra.update({k : {"connect" : "insert", "value" : v, "lang" : "/lang/en"}})""" if i['master_property']: converted_master_property = convert_name(i["master_property"], base_id, domain_id, only_include) if converted_master_property == i["master_property"]: raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \ "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"])) reciprocate_property(s, i["name"], key, converted_master_property, i["unique"], disambig=disambig, tip=tip, extra=extra) elif i['delegated']: delegate_property(s, convert_name(i['delegated'], base_id, domain_id, only_include), schema, expected=expected, tip=tip, extra=extra) else: create_property(s, i["name"], key, schema, expected, i["unique"], disambig=disambig, tip=tip, extra=extra) def _get_graph(initial_types, follow_types): """ get the graph of dependencies of all the types involved, starting with a list supplied """ assert isinstance(initial_types, (list, tuple)) graph = {} to_update = set(initial_types) done = set() while len(to_update) > 0: new = to_update.pop() graph[new] = _get_needed(s, new) if follow_types: [to_update.add(b) for b in graph[new]["__related"] if b not in done] done.update(graph[new]["__related"]) if not follow_types: # we have to check that there are no cvts attached to us, or else # ugly things happen (we can't include the cvt) for prop in graph[new]["properties"]: if prop["master_property"]: raise Exception("You can't set follow_types to False if there's a cvt. A cvt requires you get all the relevant types. Set follow_types to true.\n" + \ "The offending property was %s, whose master was %s." % (prop["id"], prop["master_property"])) return graph """def create_type_dependencies(s, base_id=None, type_id=None, follow_types=True): if base_id: q = {"id" : base_id, "/type/domain/types" : [{"id" : None}] } results = s.mqlread(q) types = map(lambda x: x["id"], results["/type/domain/types"]) #pprint(types) if type_id: types = [type_id] base_id = "/".join(type_id.split("/")[:-1]) print base_id if not base_id and not type_id: raise Exception("You need to supply either a base_id or a type_id") graph = get_graph(types, follow_types) # find distinct subgraphs (looking at needs) subgraphs = [] unknown = set(graph.keys()) if focus: unknown = set([focus]) while len(unknown) > 0: visited = set() to_visit = set([list(unknown)[0]]) while len(to_visit) > 0: new = to_visit.pop() visited.add(new) try: unknown.remove(new) except KeyError: pass [to_visit.add(b) for b in graph[new]["related"] if b not in visited] subgraphs.append(list(visited)) print "SUBGRPAHS" pprint(subgraphs) print "GRAPH" pprint(graph) return # create type dependencies typegraph = {} for tid, idres in graph.items(): typegraph[tid] = idres["needs"] least_needy_type = map(lambda (name, x): (len(x), name), typegraph.items()) least_needy_type.sort() types_to_create = create_what(least_needy_type, typegraph) # create property dependencies propgraph = {} proptotype = {} for tid, idres in graph.items(): for prop in idres["properties"]: propgraph[prop["id"]] = prop["needs"] proptotype[prop["id"]] = tid least_needy = map(lambda (name, x): (len(x), name), propgraph.items()) least_needy.sort() props_to_create = create_what(least_needy, propgraph) # CREATING STUFF domainname = "awesome" + str(int(random.random() * 1e10)) print "\ndomainid", domainname print "--------------------------" dn = s.create_private_domain(domainname, domainname + "!") domain_id = dn.domain_id better_id = s.mqlread({"id" : domain_id, "a:id" : None}) domain_id = better_id["a:id"] for type in types_to_create: print type key = "" if len(graph[type]["key"]) == 1: key = graph[type]["key"][0]["value"] else: expectedname = graph[type]["id"].split("/")[-1] if base_id: for group in graph[type]["key"]: if group["namespace"] == base_id: key = group["value"] continue if key is None: key = expectedname tip = None if graph[type]["/freebase/documented_object/tip"]: tip = graph[type]["/freebase/documented_object/tip"]["value"] create_type(s, graph[type]["name"]["value"], key, domain_id, included=map(lambda x: convert_name(x["id"], base_id, domain_id), graph[type]["/freebase/type_hints/included_types"]), cvt=graph[type]["/freebase/type_hints/mediator"], enum=graph[type]["/freebase/type_hints/enumeration"], tip=tip) print "--------------------------" for prop in props_to_create: info = graph[proptotype[prop]]["properties"] for i in info: if i["id"] == prop: schema = convert_name(proptotype[prop], base_id, domain_id) print prop expected = None if i["expected_type"]: expected = convert_name(i["expected_type"]["id"], base_id, domain_id) for k in i["key"]: if k.namespace == proptotype[prop]: key = k.value if i["/freebase/documented_object/tip"]: tip = graph[type]["/freebase/documented_object/tip"] if i['master_property']: reciprocate_property(s, i["name"], key, convert_name(i["master_property"]["id"], base_id, domain_id), i["unique"], disambig=i["/freebase/property_hints/disambiguator"], tip=tip) elif i['delegated']: delegate_property(s, convert_name(i['delegated']['id'], base_id, domain_id), schema, expected=expected, tip=tip) else: create_property(s, i["name"], key, schema, expected, i["unique"], disambig=i["/freebase/property_hints/disambiguator"], tip=tip) print "\ndomain id was", domainname """ def convert_name(old_name, operating_base, new_base, only_include=None): if old_name in only_include and old_name.startswith(operating_base): return new_base + old_name.replace(operating_base, "", 1) else: return old_name def create_what(deps, graph): create_list = [] while len(deps) > 0: neediness, id = deps.pop(0) if neediness == 0: create_list.append(id) continue else: work = True for req in graph[id]: if req not in create_list: work = False continue if work: create_list.append(id) else: deps.append((neediness, id)) return create_list def _get_needed(s, type_id): q = TYPE_QUERY q.update(id=type_id) r = s.mqlread(q) properties = r.properties # let's identify who the parent is in order to only include # other types in that domain. We don't want to go around including # all of commons because someone's a /people/person parents = [r["domain"]["id"]] included_types = map(lambda x: x["id"], r["/freebase/type_hints/included_types"]) related_types = set(included_types) for prop in properties: if prop["expected_type"]: related_types.add(prop["expected_type"]) # we have two different types of relationships: required and related. # related can be used to generate subgraphs of types # required is used to generate the dependency graph of types related = return_relevant(related_types, parents) requires = return_relevant(included_types, parents) # get property information properties = r["properties"] for prop in properties: dependent_on = set() if prop["master_property"]: dependent_on.add(prop["master_property"]) if prop["delegated"]: dependent_on.add(prop["delegated"]) prop["__requires"] = return_relevant(dependent_on, parents) # return all the information along with our special __* properties info = r info.update(__related=related, __requires=requires, __properties=properties) return info def return_relevant(start_list, parents): final = [] for item in start_list: indomain = False for parent in parents: if item.startswith(parent): indomain = True continue if indomain: final.append(item) return final if __name__ == '__main__': # create domain name = str(int(random.random()*1e10)) domain_id = s.create_private_domain("coolcat" + name, "Coolcat" + name)["domain_id"] domain_id = s.mqlread({"id" : domain_id, "a:id" : None})["a:id"] print "domain name", domain_id # dump information #graph = dump_type(s, "/user/nitromaster101/coolcat3194242218/bridge_player", follow_types=True) graph = dump_type(s, "/people/person", True) # upload it somewhere else upload_type(s, graph, domain_id, debug=True) print "domain name", domain_id #create_type_dependencies(s, type_id="/base/contractbridge/bridge_player")