diff options
| -rw-r--r-- | freebase/api/session.py | 46 | ||||
| -rw-r--r-- | freebase/schema.py | 3 | ||||
| -rw-r--r-- | linkmerge.py | 108 | ||||
| -rw-r--r-- | merge.py | 121 | ||||
| -rw-r--r-- | split.py | 122 |
5 files changed, 379 insertions, 21 deletions
diff --git a/freebase/api/session.py b/freebase/api/session.py index 609a64f..2348579 100644 --- a/freebase/api/session.py +++ b/freebase/api/session.py @@ -39,7 +39,7 @@ declarations for external metaweb api. __all__ = ['MetawebError', 'MetawebSession', 'HTTPMetawebSession', 'attrdict'] -__version__ = '1.0' +__version__ = '1.01' import os, sys, re import cookielib @@ -339,12 +339,11 @@ class HTTPMetawebSession(MetawebSession): headerstr = '\nHEADERS:\n ' + '\n '.join([('%s: %s' % (k,v)) for k,v in headers.items()]) self.log.info('%s %s%s%s', method, url, formstr, headerstr) - ####### # just in case you decide to make SUPER ridiculous GET queries: if len(url) > 1000 and method == "GET": method = "POST" - url, body = url.split("?") + url, body = url.split("?", 1) ct = 'application/x-www-form-urlencoded' headers['content-type'] = ct + '; charset=utf-8' @@ -631,12 +630,14 @@ class HTTPMetawebSession(MetawebSession): return body - def mqlwrite(self, sq, use_permission_of=None): + def mqlwrite(self, sq, use_permission_of=None, attribution_id=None): """do a mql write. For a more complete description, see http://www.freebase.com/view/en/api_service_mqlwrite""" query = dict(query=sq, escape=False) if use_permission_of: query['use_permission_of'] = use_permission_of + if attribution_id: + query['attribution'] = attribution_id qstr = json.dumps(query, separators=SEPARATORS) @@ -867,16 +868,18 @@ class HTTPMetawebSession(MetawebSession): ### SCHEMA MANIPULATION ### # Object helpers - def create_object(self, name="", path=None, key=None, namespace=None, included_types=None, create="unless_exists", extra=None): + def create_object(self, name="", path=None, key=None, namespace=None, included_types=None, create="unless_exists", extra=None, use_permission_of=None, attribution=None): if type(included_types) is str: included_types = [included_types] if path and (key or namespace): - raise Exception("You can't specify both the path and a key and namespace.") + raise ValueError("You can't specify both the path and a key and namespace.") if path: key, namespace = get_key_namespace(path) + if (key and not namespace) or (not key and namespace): + raise ValueError("You must either specify both a key and a namespace, or neither.") if included_types: its = set(included_types) @@ -890,23 +893,26 @@ class HTTPMetawebSession(MetawebSession): wq = { "id" : None, "name" : name, - "key" : { - "namespace" : namespace, - "value" : key, - }, "create" : create } + # conditionally add key creation + if key: + wq.update({"key" : { + "namespace" : namespace, + "value" : key, + }}) + if included_types: wq.update(type = [{ "id" : it, "connect" : "insert" } for it in its]) if extra: wq.update(extra) - return self.mqlwrite(wq) + return self.mqlwrite(wq, use_permission_of=use_permission_of, attribution_id=attribution) - def connect_object(self, id, newpath, extra=None): + def connect_object(self, id, newpath, extra=None, use_permission_of=None, attribution=None): key, namespace = get_key_namespace(newpath) @@ -921,10 +927,10 @@ class HTTPMetawebSession(MetawebSession): if extra: wq.update(extra) - return self.mqlwrite(wq) + return self.mqlwrite(wq, use_permission_of=use_permission_of, attribution_id=attribution) - def disconnect_object(self, id, extra=None): + def disconnect_object(self, id, extra=None, use_permission_of=None, attribution=None): key, namespace = get_key_namespace(id) @@ -937,11 +943,11 @@ class HTTPMetawebSession(MetawebSession): } } if extra: wq.update(extra) - return self.mqlwrite(wq) + return self.mqlwrite(wq, use_permission_of=use_permission_of, attribution_id=attribution) - def move_object(self, oldpath, newpath): - a = self.connect_object(oldpath, newpath) - b = self.disconnect_object(oldpath) + def move_object(self, oldpath, newpath, use_permission_of=None, attribution=None): + a = self.connect_object(oldpath, newpath, use_permission_of=use_permission_of, attribution=attribution) + b = self.disconnect_object(oldpath, use_permission_of=use_permission_of, attribution=attribution) return a, b @@ -958,8 +964,8 @@ if __name__ == '__main__': mss = HTTPMetawebSession('sandbox.freebase.com') - self.mss.log.setLevel(logging.DEBUG) - self.mss.log.addHandler(console) + mss.log.setLevel(logging.DEBUG) + mss.log.addHandler(console) print mss.mqlread([dict(name=None, type='/type/type')]) diff --git a/freebase/schema.py b/freebase/schema.py index 17f023c..01d6074 100644 --- a/freebase/schema.py +++ b/freebase/schema.py @@ -51,7 +51,7 @@ def key_exists(s, k): def type_object(s, id, type_id): q = { "id" : type_id, - "/freebase/type_hints/included_types" : [{"id" : None}] + "/freebase/type_hints/included_types" : [{"id" : None, "optional" : True}] } included_types = map(lambda x: x["id"], s.mqlread(q)["/freebase/type_hints/included_types"]) @@ -136,6 +136,7 @@ def get_property_info(s, prop_id): # Create Type def create_type(s, name, key, ns, cvt=False, tip=None, included=None, extra=None): + # TODO: CREATE SYNTHETIC VIEW if key_exists(s, ns + "/" + key ): return diff --git a/linkmerge.py b/linkmerge.py new file mode 100644 index 0000000..3cfe1f1 --- /dev/null +++ b/linkmerge.py @@ -0,0 +1,108 @@ + +import freebase, freebase.schema +from freebase.api import LITERAL_TYPE_IDS, MetawebError + +from copy import deepcopy +import itertools +import logging + +ALL_LINKS_QUERY = [{"type": "/type/link", + "source": {"id" : None}, + "target": {"id" : None}, + "target_value": None, + "master_property": { "id" : None, "expected_type" : None, "unique" : None }, + "operation": None, + "valid": True }] + +def merge(s, amoeba_id, target_id): + # In merging, we'll use the analogy of phagocytosis. + # http://en.wikipedia.org/wiki/Phagocytosis + # In this example, the amoeba is the main guy who is swallowing the target + + # In cases where there is no merging problem, it doesn't matter who is the + # amoeba and who is the target, but the final merge product will be in the amoeba + + # this merging will be done using links + # effectively, we want to move everything that links to the target + # and link it to the amoeba. This does present some issues: + # some things just can't be moved (/en keys, for example). + + target_source_l, target_target_l, target_target_v_l = get_all_links(target_id) + amoeba_source_l, amoeba_target_l, amoeba_target_v_l = get_all_links(amoeba_id) + + total_delete_query = {} + total_write_query = { "id" : amoeba_id } + + # let's redirect all source and target links on target to amoeba + #print [i for i in target_source_l]; print + for link in target_source_l: + # try this + if link.master_property.expected_type != "/type/text" and \ + link.master_property.expected_type != "/type/key" and \ + link.master_property.id != "/type/object/permission" and \ + (not (link.master_property.unique and exists(amoeba_source_l, amoeba_target_l, link.master_property.id))): + prop = link.master_property.id + print prop + current_prop = total_write_query.get(prop, []) + current_prop.append({"id" : link.target.id, "connect" : "replace"}) + total_write_query[prop] = current_prop + + to_delete_prop = total_delete_query.get(prop, []) + to_delete_prop.append({"id" : link.target.id, "connect" : "delete"}) + total_delete_query[prop] = to_delete_prop + + for link in target_target_l: + if link.master_property.expected_type != "/type/text" and \ + link.master_property.expected_type != "/type/key" and \ + link.master_property.id != "/type/object/permission": + + prop = "!" + link.master_property.id + print prop + current_prop = total_write_query.get(prop, []) + current_prop.append({"id" : link.source.id, "connect" : "replace"}) + total_write_query[prop] = current_prop + + to_delete_prop = total_delete_query.get(prop, []) + to_delete_prop.append({"id" : link.source.id, "connect" : "delete"}) + total_delete_query[prop] = to_delete_prop + + # delete old + badpropnames = set([]) + for propname, guys in total_delete_query.iteritems(): + new = dict({propname:guys, "id":target_id}) + try: + s.mqlwrite(new) + except MetawebError, me: + print "Oh well, %s failed. %s" % (propname, me) + badpropnames.add(propname) + for badprop in badpropnames: + del total_write_query[badprop] + s.mqlwrite(total_write_query) + + +def exists(source_links, target_links, property_id): + print "testing", property_id + for link in itertools.chain(source_links, target_links): + if link.master_property.id == property_id: + if link.target and link.source: + print "outta here", property_id + return True + return False + return False + + + +def get_all_links(the_id): + source, target, target_value = [deepcopy(ALL_LINKS_QUERY) for i in range(3)] + source[0].update(source={"id" : the_id}) + target[0].update(target={"id" : the_id}) + target_value[0].update(target_value={"id" : the_id }) + + return (s.mqlreaditer(source), + s.mqlreaditer(target), + s.mqlreaditer(target_value)) + + +if __name__ == '__main__': + s = freebase.api.HTTPMetawebSession("http://sandbox-freebase.com") + merge(s, "/guid/9202a8c04000641f800000000bc3141d", "/guid/9202a8c04000641f800000000aa5533e")
\ No newline at end of file diff --git a/merge.py b/merge.py new file mode 100644 index 0000000..3e9dd04 --- /dev/null +++ b/merge.py @@ -0,0 +1,121 @@ +import freebase, freebase.schema +from freebase.api import LITERAL_TYPE_IDS + +import logging + +def merge(s, amoeba_id, target_id): + # We'll merge by types. This isn't really an issue, since everything + # displayed in the UI is by types + + # In merging, we'll use the analogy of phagocytosis. + # http://en.wikipedia.org/wiki/Phagocytosis + # In this example, the amoeba is the main guy who is swallowing the target + + # types_to_merge = set(["/common/topic"]) let's merge all for now. + + # In cases where there is no merging problem, it doesn't matter who is the + # amoeba and who is the target, but the final merge product will be in the amoeba + + # get all the properties of amoeba+target + amoeba_types = get_types(amoeba_id) + target_types = get_types(target_id) + all_types = amoeba_types.union(target_types) + + properties_unique = {} + properties_expected = {} + type_to_properties = {} + + for type_id in all_types: + unique_property_query = { "id" : type_id, + "type" : "/type/type", + "properties" : [{ + "id" : None, + "unique" : None, + "expected_type" : None + }] } + r = s.mqlread(unique_property_query) + + all_properties = [] + if r: + for prop in r["properties"]: + properties_unique[prop.id] = prop.unique + properties_expected[prop.id] = prop.expected_type + all_properties.append(prop) + type_to_properties[type_id] = all_properties + + # type amoeba with new types in target + for type_id in target_types: + freebase.schema.type_object(s, amoeba_id, type_id) + + # get all properties of target + mega_target_query = { "id" : target_id } + mega_amoeba_query = { "id" : amoeba_id } + for type_id in target_types: + for prop_id in type_to_properties[type_id]: + mega_target_query.update({prop_id["id"]:[{}]}) + for type_id in amoeba_types: + for prop_id in type_to_properties[type_id]: + mega_amoeba_query.update({prop_id["id"]:[{}]}) + + + # for every non-empty property in target: + # 1. if it doesn't exist in amoeba, add replace-style + # 2. if it does exist in amoeba: if the property is unique, do nothing + # if property is not unique, just add replace + + target_result, amoeba_result = s.mqlreadmulti([mega_target_query, mega_amoeba_query]) + + property_values = {} + for prop, value in target_result.iteritems(): + if prop in properties_unique.iterkeys(): + # if value is primitive + if properties_expected[prop] in LITERAL_TYPE_IDS: + property_values[prop] = [{"value" : b["value"]} + for b in value] + else: + property_values[prop] = [{"id" : b["id"]} + for b in value] + + master_write_amoeba_query = { "id" : amoeba_id } + for prop, val in property_values.iteritems(): + if val: + if amoeba_result.has_key(prop) and amoeba_result[prop]: + # if property is unique, do nothing + # if property is not unique, just add replace + if not properties_unique[prop]: + [b.update(connect="replace") for b in val] + master_write_amoeba_query.update({prop:val}) + + else: + [b.update(connect="replace") for b in val] + master_write_amoeba_query.update({prop:val}) + + # delete target information + + + # write amoeba information + s.mqlwrite(master_write_amoeba_query) + + # make name of target an alias in amoeba + make_alias_query = {"id" : amoeba_id, + "alias" : dddddddddddddddddd} + + # migrate data (thinks linking here) + # get all the links from the target to someone else + +def get_types(topic_id): + type_query = {"id" : topic_id, "type" : [{"id" : None}]} + return set([type_obj["id"] for type_obj in s.mqlread(type_query)["type"]]) + + +if __name__ == '__main__': + s = freebase.api.HTTPMetawebSession("http://sandbox-freebase.com") + + """console = logging.StreamHandler() + console.setLevel(logging.DEBUG) + + s.log.setLevel(logging.DEBUG) + s.log.addHandler(console)""" + + merge(s, "/guid/9202a8c04000641f800000000bc3141d", "/guid/9202a8c04000641f800000000aa5533e") + #merge(s, "/en/the_beatles", "/en/the_police")
\ No newline at end of file diff --git a/split.py b/split.py new file mode 100644 index 0000000..a8d0310 --- /dev/null +++ b/split.py @@ -0,0 +1,122 @@ +import freebase, freebase.schema +from freebase.api.session import LITERAL_TYPE_IDS + +from copy import deepcopy + +## TODO: CLONE TYPES + +class AttributionNode(object): + def __init__(self, s): + self._dict = {} + self.s = s + + def get(self, user): + if self._dict.has_key(user): + return self._dict[user] + + # create attribution node + attribution_id = self.s.mqlwrite({"create" : "unconditional", + "type": "/type/attribution", + "id" : None })["id"] + self._dict[user] = attribution_id + return self._dict[user] + + def set(self, user, attribution_id): + self._dict[user] = attribution_id + + +s = freebase.api.HTTPMetawebSession("http://sandbox-freebase.com") + +# let's determine the split locations +topic_id = "/en/the_beatles" + +a = AttributionNode(s) + +# get all types +type_query = {"id" : topic_id, "type" : [{"id" : None}]} +types = set([type_id["id"] for type_id in s.mqlread(type_query)["type"]]) + +split = set(["/music/artist"]) +keep = types.difference(split) + +properties_expected = {} +type_to_properties = {} + +for type_id in split: + expected_type_property_query = { "id" : type_id, + "type" : "/type/type", + "properties" : [{ + "id" : None, + "expected_type" : None + }]} + r = s.mqlread(expected_type_property_query) + all_properties = [] + if r: + for prop in r["properties"]: + prop_id = prop["id"] + prop_expected_type = prop["expected_type"] + properties_expected[prop_id] = prop_expected_type + all_properties.append(prop) + type_to_properties[type_id] = all_properties + +# split + +## create new object with new types and correct attribution +newname = s.mqlread({"id" : topic_id, "name" : None})["name"] +#attribution = s.mqlread({"id" : topic_id, "attribution" : None})["attribution"] +user_id = s.user_info()["id"] +attribution = a.get(user_id) + +new_object_id = s.create_object(newname, included_types=list(split), + create="unconditional", + attribution=attribution)["id"] + +# import data from old +# if expected_type is primitive (in LITERAL_TYPE_IDS), then we look for value. +# Else, we look for id + +mega_query = { "id" : topic_id } +for prop_id in properties_expected.iterkeys(): + mega_query.update({prop_id:[{}]}) + +res = s.mqlread(mega_query) + +property_values = {} +for prop, value in res.iteritems(): + if prop in properties_expected.iterkeys(): + # if value is primitive + if properties_expected[prop] in LITERAL_TYPE_IDS: + property_values[prop] = [{"value" : b["value"]} + for b in res[prop]] + else: + property_values[prop] = [{"id" : b["id"]} + for b in res[prop]] + +master_write_query = { "id" : new_object_id } +master_delete_query = { "id" : topic_id } +for prop, val in property_values.iteritems(): + if val: + [v.update(connect="replace") for v in val] + master_write_query.update({prop:val}) + deleteval = deepcopy(val) + [dv.update(connect="delete") for dv in deleteval] + master_delete_query.update({prop:deleteval}) + +# before we write, we have to delete all the old information +# this is because we don't can't have two similar guys connecting +# to the same cvt + +# remove types (and properties from old) +# delete types +# (we can't get rid of included_types easily... not sure who depends on whom) +delete_types_query = { "id" : topic_id, + "type":[{"id" : type_id, "connect" : "delete"} + for type_id in split]} +s.mqlwrite(delete_types_query) + +# delete properties +s.mqlwrite(master_delete_query) + +# add the data to the new guy +s.mqlwrite(master_write_query) +print "new object was", new_object_id |