cleanup, incorporate changes from code review, and move merge and split stuff into freebase.experimental

git-svn-id: http://freebase-python.googlecode.com/svn/trunk@201 5914aa95-5b3a-0410-a3b5-7b719e7fe9b2

3 files changed, 362 insertions, 0 deletions

diff --git a/freebase/experimental/linkmerge.py b/freebase/experimental/linkmerge.py
new file mode 100644
index 0000000..3cfe1f1
--- /dev/null
+++ b/freebase/experimental/linkmerge.py
@@ -0,0 +1,108 @@
+
+import freebase, freebase.schema
+from freebase.api import LITERAL_TYPE_IDS, MetawebError
+
+from copy import deepcopy
+import itertools
+import logging
+
+ALL_LINKS_QUERY = [{"type": "/type/link",
+                    "source": {"id" : None},
+                    "target": {"id" : None},
+                    "target_value": None,
+                    "master_property": { "id" : None, "expected_type" : None, "unique" : None },
+                    "operation": None,
+                    "valid": True }]
+
+def merge(s, amoeba_id, target_id):
+    # In merging, we'll use the analogy of phagocytosis.
+    # http://en.wikipedia.org/wiki/Phagocytosis
+    # In this example, the amoeba is the main guy who is swallowing the target
+
+    # In cases where there is no merging problem, it doesn't matter who is the
+    # amoeba and who is the target, but the final merge product will be in the amoeba
+
+    # this merging will be done using links
+    # effectively, we want to move everything that links to the target
+    # and link it to the amoeba. This does present some issues:
+    # some things just can't be moved (/en keys, for example).
+    
+    target_source_l, target_target_l, target_target_v_l = get_all_links(target_id)
+    amoeba_source_l, amoeba_target_l, amoeba_target_v_l = get_all_links(amoeba_id)
+
+    total_delete_query = {}
+    total_write_query = { "id" : amoeba_id }
+
+    # let's redirect all source and target links on target to amoeba
+    #print [i for i in target_source_l]; print
+    for link in target_source_l:
+        # try this
+        if link.master_property.expected_type != "/type/text" and \
+            link.master_property.expected_type != "/type/key" and \
+            link.master_property.id != "/type/object/permission" and \
+            (not (link.master_property.unique and exists(amoeba_source_l, amoeba_target_l, link.master_property.id))):
+            prop = link.master_property.id
+            print prop
+            current_prop = total_write_query.get(prop, [])
+            current_prop.append({"id" : link.target.id, "connect" : "replace"})
+            total_write_query[prop] = current_prop
+            
+            to_delete_prop = total_delete_query.get(prop, [])
+            to_delete_prop.append({"id" : link.target.id, "connect" : "delete"})
+            total_delete_query[prop] = to_delete_prop
+            
+    for link in target_target_l:
+        if link.master_property.expected_type != "/type/text" and \
+            link.master_property.expected_type != "/type/key" and \
+            link.master_property.id != "/type/object/permission":
+            
+            prop = "!" + link.master_property.id
+            print prop
+            current_prop = total_write_query.get(prop, [])
+            current_prop.append({"id" : link.source.id, "connect" : "replace"})
+            total_write_query[prop] = current_prop
+            
+            to_delete_prop = total_delete_query.get(prop, [])
+            to_delete_prop.append({"id" : link.source.id, "connect" : "delete"})
+            total_delete_query[prop] = to_delete_prop
+    
+    # delete old
+    badpropnames = set([])
+    for propname, guys in total_delete_query.iteritems():
+        new = dict({propname:guys, "id":target_id})
+        try:
+            s.mqlwrite(new)
+        except MetawebError, me:
+            print "Oh well, %s failed. %s" % (propname, me)
+            badpropnames.add(propname)
+    for badprop in badpropnames:
+        del total_write_query[badprop]
+    s.mqlwrite(total_write_query)
+        
+    
+def exists(source_links, target_links, property_id):
+    print "testing", property_id
+    for link in itertools.chain(source_links, target_links):
+        if link.master_property.id == property_id:
+            if link.target and link.source:
+                print "outta here", property_id
+                return True
+            return False
+    return False
+    
+
+
+def get_all_links(the_id):
+    source, target, target_value = [deepcopy(ALL_LINKS_QUERY) for i in range(3)]
+    source[0].update(source={"id" : the_id})
+    target[0].update(target={"id" : the_id})
+    target_value[0].update(target_value={"id" : the_id })
+
+    return (s.mqlreaditer(source),
+            s.mqlreaditer(target),
+            s.mqlreaditer(target_value))
+
+
+if __name__ == '__main__':
+    s = freebase.api.HTTPMetawebSession("http://sandbox-freebase.com")
+    merge(s, "/guid/9202a8c04000641f800000000bc3141d", "/guid/9202a8c04000641f800000000aa5533e")
\ No newline at end of file
diff --git a/freebase/experimental/merge.py b/freebase/experimental/merge.py
new file mode 100644
index 0000000..0d7ddf7
--- /dev/null
+++ b/freebase/experimental/merge.py
@@ -0,0 +1,132 @@
+import freebase, freebase.schema
+from freebase.api import LITERAL_TYPE_IDS
+
+import logging
+
+def merge(s, amoeba_id, target_id):
+    # We'll merge by types. This isn't really an issue, since everything
+    # displayed in the UI is by types
+
+    # In merging, we'll use the analogy of phagocytosis.
+    # http://en.wikipedia.org/wiki/Phagocytosis
+    # In this example, the amoeba is the main guy who is swallowing the target
+
+    # types_to_merge = set(["/common/topic"]) let's merge all for now.
+
+    # In cases where there is no merging problem, it doesn't matter who is the
+    # amoeba and who is the target, but the final merge product will be in the amoeba
+
+    # get all the properties of amoeba+target
+    amoeba_types = get_types(amoeba_id)
+    target_types = get_types(target_id)
+    all_types = amoeba_types.union(target_types)
+
+    properties_unique = {}
+    properties_expected = {}
+    type_to_properties = {}
+
+    for type_id in all_types:
+        unique_property_query = { "id" : type_id, 
+                                   "type" : "/type/type",
+                                   "properties" : [{
+                                        "id" : None, 
+                                        "unique" : None,
+                                        "expected_type" : None
+                                    }] }
+        r = s.mqlread(unique_property_query)
+        
+        all_properties = []
+        if r:
+            for prop in r["properties"]:
+                properties_unique[prop.id] = prop.unique
+                properties_expected[prop.id] = prop.expected_type
+                all_properties.append(prop)
+        type_to_properties[type_id] = all_properties
+
+    # type amoeba with new types in target
+    for type_id in target_types:
+        freebase.schema.add_type_to_object(s, amoeba_id, type_id)
+
+    # get all properties of target
+    mega_target_query = { "id" : target_id }
+    mega_amoeba_query = { "id" : amoeba_id }
+    for type_id in target_types:
+        for prop_id in type_to_properties[type_id]:
+            mega_target_query.update({prop_id["id"]:[{}]})
+    for type_id in amoeba_types:
+        for prop_id in type_to_properties[type_id]:
+            mega_amoeba_query.update({prop_id["id"]:[{}]})
+        
+
+    # for every non-empty property in target:
+    #  1. if it doesn't exist in amoeba, add replace-style
+    #  2. if it does exist in amoeba:  if the property is unique, do nothing
+    #                                  if property is not unique, just add replace
+
+    target_result, amoeba_result = s.mqlreadmulti([mega_target_query, mega_amoeba_query])
+
+    property_values = {}
+    for prop, value in target_result.iteritems():
+        if prop in properties_unique.iterkeys():
+            # if value is primitive
+            if properties_expected[prop] in LITERAL_TYPE_IDS:
+                property_values[prop] = [{"value" : b["value"]} 
+                                            for b in value]
+            else:
+                property_values[prop] = [{"id" : b["id"]} 
+                                            for b in value]
+    
+    master_write_amoeba_query = { "id" : amoeba_id }
+    for prop, val in property_values.iteritems():
+        if val:
+            if amoeba_result.has_key(prop) and amoeba_result[prop]:
+                # if property is unique, do nothing
+                # if property is not unique, just add replace
+                if not properties_unique[prop]:
+                    [b.update(connect="replace") for b in val]
+                    master_write_amoeba_query.update({prop:val})
+                
+            else:
+                [b.update(connect="replace") for b in val]
+                master_write_amoeba_query.update({prop:val})
+    
+    # delete target information
+    
+        
+    # write amoeba information
+    s.mqlwrite(master_write_amoeba_query)
+    
+    # make name of target an alias in amoeba
+    get_amoeba_name_query = {"id" : amoeba_id,
+                             "name" : None }
+    get_target_alias_query = {"id" : target_id,
+                              "/common/topic/alias" : [{}] }
+    amoeba_name, target_aliases = s.mqlread([get_amoeba_name_query,
+                                            get_target_alias_query])
+    # TODO:
+    #set_alias_query = {"id" : amoeba_id}
+    #for alias in target_aliases["/common/topic/alias"]:
+    #    if alias.value != amoeba_name:
+    #        set_alias_query.update({"lang": "/lang/en",
+    #        "value": "Beeetles",
+    #        "connect": "insert"})
+    
+    # migrate data (thinks linking here)
+    # get all the links from the target to someone else
+
+def get_types(topic_id):
+    type_query = {"id" : topic_id, "type" : [{"id" : None}]}
+    return set([type_obj["id"] for type_obj in s.mqlread(type_query)["type"]])
+    
+
+if __name__ == '__main__':
+    s = freebase.api.HTTPMetawebSession("http://sandbox-freebase.com")
+    
+    """console = logging.StreamHandler()
+    console.setLevel(logging.DEBUG)
+
+    s.log.setLevel(logging.DEBUG)
+    s.log.addHandler(console)"""
+
+    merge(s, "/guid/9202a8c04000641f800000000bc3141d", "/guid/9202a8c04000641f800000000aa5533e")
+    #merge(s, "/en/the_beatles", "/en/the_police")
\ No newline at end of file
diff --git a/freebase/experimental/split.py b/freebase/experimental/split.py
new file mode 100644
index 0000000..a8d0310
--- /dev/null
+++ b/freebase/experimental/split.py
@@ -0,0 +1,122 @@
+import freebase, freebase.schema
+from freebase.api.session import LITERAL_TYPE_IDS
+
+from copy import deepcopy
+
+## TODO: CLONE TYPES
+
+class AttributionNode(object):
+    def __init__(self, s):
+        self._dict = {}
+        self.s = s
+        
+    def get(self, user):
+        if self._dict.has_key(user):
+            return self._dict[user]
+        
+        # create attribution node
+        attribution_id = self.s.mqlwrite({"create" : "unconditional", 
+                            "type": "/type/attribution",
+                            "id" : None })["id"]
+        self._dict[user] = attribution_id
+        return self._dict[user]
+    
+    def set(self, user, attribution_id):
+        self._dict[user] = attribution_id
+        
+
+s = freebase.api.HTTPMetawebSession("http://sandbox-freebase.com")
+
+# let's determine the split locations
+topic_id = "/en/the_beatles"
+
+a = AttributionNode(s)
+
+# get all types
+type_query = {"id" : topic_id, "type" : [{"id" : None}]}
+types = set([type_id["id"] for type_id in s.mqlread(type_query)["type"]])
+
+split = set(["/music/artist"])
+keep = types.difference(split)
+
+properties_expected = {}
+type_to_properties = {}
+
+for type_id in split:
+    expected_type_property_query = { "id" : type_id, 
+                                     "type" : "/type/type",
+                                     "properties" : [{
+                                         "id" : None, 
+                                         "expected_type" : None
+                                     }]}
+    r = s.mqlread(expected_type_property_query)
+    all_properties = []
+    if r:
+        for prop in r["properties"]:
+            prop_id = prop["id"]
+            prop_expected_type = prop["expected_type"]
+            properties_expected[prop_id] = prop_expected_type
+            all_properties.append(prop)
+    type_to_properties[type_id] = all_properties
+
+# split
+
+## create new object with new types and correct attribution
+newname = s.mqlread({"id" : topic_id, "name" : None})["name"]
+#attribution = s.mqlread({"id" : topic_id, "attribution" : None})["attribution"]
+user_id = s.user_info()["id"]
+attribution = a.get(user_id)
+
+new_object_id = s.create_object(newname, included_types=list(split), 
+                                create="unconditional",
+                                attribution=attribution)["id"]
+
+# import data from old
+# if expected_type is primitive (in LITERAL_TYPE_IDS), then we look for value.
+# Else, we look for id
+
+mega_query = { "id" : topic_id }
+for prop_id in properties_expected.iterkeys():
+    mega_query.update({prop_id:[{}]})
+
+res = s.mqlread(mega_query)
+
+property_values = {}
+for prop, value in res.iteritems():
+    if prop in properties_expected.iterkeys():
+        # if value is primitive
+        if properties_expected[prop] in LITERAL_TYPE_IDS:
+            property_values[prop] = [{"value" : b["value"]} 
+                                        for b in res[prop]]
+        else:
+            property_values[prop] = [{"id" : b["id"]} 
+                                        for b in res[prop]]
+
+master_write_query = { "id" : new_object_id }
+master_delete_query = { "id" : topic_id }
+for prop, val in property_values.iteritems():
+    if val:
+        [v.update(connect="replace") for v in val]
+        master_write_query.update({prop:val})
+        deleteval = deepcopy(val)
+        [dv.update(connect="delete") for dv in deleteval]
+        master_delete_query.update({prop:deleteval})
+
+# before we write, we have to delete all the old information
+# this is because we don't can't have two similar guys connecting
+# to the same cvt
+
+# remove types (and properties from old)
+# delete types
+# (we can't get rid of included_types easily... not sure who depends on whom)
+delete_types_query = { "id" : topic_id, 
+                       "type":[{"id" : type_id, "connect" : "delete"}
+                            for type_id in split]}
+s.mqlwrite(delete_types_query)
+
+# delete properties
+s.mqlwrite(master_delete_query)
+
+# add the data to the new guy
+s.mqlwrite(master_write_query)
+print "new object was", new_object_id