From 68b2334cd8484ac72814060ced5703a2f94b57d7 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 23 Sep 2014 16:28:16 +0300 Subject: initial implementation --- src-py/neo4j_util.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 src-py/neo4j_util.py (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py new file mode 100644 index 00000000..009fbe5f --- /dev/null +++ b/src-py/neo4j_util.py @@ -0,0 +1,49 @@ +""" + Utility code in speaking the neo4j REST api +""" + +import json +import urllib2 + +def post_neo4j(url, data): + """ + return dict translation of the json string returned by neo4j, raise Exception if the 'errors' key is not empty + """ + ret = post(url, data) + ret_data = json.load(ret) + + if ret_data['errors']: + raise Exception('neo4j exception: ' + str(ret_data['errors'])) + + return ret_data + +def post(url, data): + assert(isinstance(data, dict)) # make sure we're not handed json strings + + post_data_json = json.dumps(data) + + req = urllib2.Request(url) + req.add_header('User-Agent', 'rhizi-server/0.1') + req.add_header('Accept', 'application/json; charset=UTF-8') + req.add_header('Content-Type', 'application/json') + + try: + ret = urllib2.urlopen(req, post_data_json) + except urllib2.HTTPError as e: + raise Exception('post request failed: code: {0}, reason: {1}'.format(e.code, e.reason)) + + return ret + +def statement_to_REST_form(query, parameters={}): + """ + turn cypher query to neo4j json API format + """ + assert isinstance(query, str) + assert isinstance(parameters, dict) + + return {'statement' : query, 'parameters': parameters} + +def statement_set_to_REST_form(statement_set): + assert isinstance(statement_set, list) + + return {'statements': statement_set} -- cgit v1.3.1 From c4dab7675a438781a36fb70e769ed7b911f9ef25 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 28 Sep 2014 23:50:06 +0300 Subject: type check against 'basestring' rather than str, tagging with: [python2] --- src-py/neo4j_util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 009fbe5f..4c166bfe 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -7,7 +7,8 @@ import urllib2 def post_neo4j(url, data): """ - return dict translation of the json string returned by neo4j, raise Exception if the 'errors' key is not empty + @return dict object from the neo4j json POST response + @raise exception: if the 'errors' key is not empty """ ret = post(url, data) ret_data = json.load(ret) @@ -38,7 +39,7 @@ def statement_to_REST_form(query, parameters={}): """ turn cypher query to neo4j json API format """ - assert isinstance(query, str) + assert isinstance(query, basestring) assert isinstance(parameters, dict) return {'statement' : query, 'parameters': parameters} -- cgit v1.3.1 From eef0c89ed89293ca89ba8b5f42b7100c46682fd7 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 6 Oct 2014 18:16:12 +0300 Subject: filter_attr_map to cypher where clause conversion: where_clause_from_filter_attr_map() --- src-py/neo4j_util.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 4c166bfe..929324fb 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -48,3 +48,19 @@ def statement_set_to_REST_form(statement_set): assert isinstance(statement_set, list) return {'statements': statement_set} + +def where_clause_from_filter_attr_map(filter_attr_map, node_param_name="n"): + """ + convert a filter attribute map to a parameterized Cypher where clause, eg. + in: { 'att_foo': [ 'a', 'b' ], 'att_goo': [1,2] } + out: where n.att_foo in {att_foo} and n.att_goo in {att_goo} ... + """ + filter_arr = [] + for k in filter_attr_map.keys(): + # create a cypher query parameter place holder for each attr set + # eg. n.foo in {foo}, where foo is passed as a query parameter + f_attr = "{0}.{1} in {{{1}}}".format(node_param_name, k) + filter_arr.append(f_attr) + filter_str = "where {0}".format(' and '.join(filter_arr)) + return filter_str + -- cgit v1.3.1 From e4fbde3cea8a5febda828b12927a3759d430606d Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 6 Oct 2014 18:49:14 +0300 Subject: handle empty filter maps --- src-py/db_controller.py | 42 ++++++++++++++++++++++-------------------- src-py/neo4j_util.py | 5 +++++ 2 files changed, 27 insertions(+), 20 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index b4ce0fc5..dcc22f7c 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -112,7 +112,7 @@ class DBO_add_node_set(DB_op): class DBO_load_node_id_set(DB_op): def __init__(self, filter_type, filter_prop=None): - # TODO: mv type filter to DBO_load_node_set_by_attribute + # TODO: mv type filter to DBO_load_node_set """ load node DB id set, filter by type / properties """ @@ -161,35 +161,23 @@ class DBO_load_node_set_by_DB_id(DB_op): log.debug('loaded node set: ' + str(data)) return self.extract_single_query_response_data(self.statement_set[0], data) -class DBO_load_node_set_by_attribute(DB_op): +class DBO_load_node_set(DB_op): - def __init__(self, filter_attr_map): + def __init__(self, filter_type=None, filter_attr_map=None): """ load a set of nodes according to filter_attr_map @param filter_attr_map: is a filter_key to filter_value_set map of attributes to match against, eg.: { 'id':[0,1], 'color: ['red','blue'] } + @param filter_type: node type filter @return: loaded node set or an empty set if no match was found """ - # type sanity checks - assert isinstance(filter_attr_map, dict) - assert len(filter_attr_map) > 0 - for k, v in filter_attr_map.items(): - assert isinstance(k, basestring) - assert isinstance(v, list) - - filter_arr = [] - for k, v in filter_attr_map.items(): - # create a cypher query parameter place holder for each attr set - # eg. n.foo in {foo}, where foo is passed as a query parameter - f_attr = "n.{0} in {{{0}}}".format(k, v) - filter_arr.append(f_attr) + self.__type_check_filter_attr_map(filter_attr_map) + filter_str = dbu.where_clause_from_filter_attr_map() - filter_str = "where {0}".format(' and '.join(filter_arr)) - - super(DBO_load_node_set_by_attribute, self).__init__() + super(DBO_load_node_set, self).__init__() q = "match (n) {0} return n".format(filter_str) self.add_statement(q, params=filter_attr_map) @@ -197,7 +185,7 @@ class DBO_load_node_set_by_attribute(DB_op): log.debug('loaded node set: ' + str(data)) return self.extract_single_query_response_data(self.statement_set[0], data) -class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): +class DBO_load_node_set_by_id_attribute(DBO_load_node_set): def __init__(self, id_set): """ convenience op: load a set of nodes by their 'id' attribute != DB node id @@ -206,6 +194,20 @@ class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): super(DBO_load_node_set_by_id_attribute, self).__init__({'id': id_set}) +class DBO_load_link_id_set(DB_op): + def __init__(self, filter_type=None, filter_attr_map=None): + """ + load a set of link ids + + @param filter_type: link type filter + @param filter_attr_map: is a filter_key to filter_value_set map of + attributes to match link properties against + @return: a set of loaded link ids + """ + self.__type_check_filter_attr_map(filter_attr_map) + filter_str = dbu.where_clause_from_filter_attr_map() + + class DB_Driver_Base(): pass diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 929324fb..c340e15a 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -54,7 +54,12 @@ def where_clause_from_filter_attr_map(filter_attr_map, node_param_name="n"): convert a filter attribute map to a parameterized Cypher where clause, eg. in: { 'att_foo': [ 'a', 'b' ], 'att_goo': [1,2] } out: where n.att_foo in {att_foo} and n.att_goo in {att_goo} ... + + @param filter_attr_map: may be None or empty """ + if not filter_attr_map: + return "" + filter_arr = [] for k in filter_attr_map.keys(): # create a cypher query parameter place holder for each attr set -- cgit v1.3.1 From 4717f01ed33b26f6c0fbbf2cb83fbf681d14da41 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 7 Oct 2014 18:08:20 +0200 Subject: mv type sanity checks to neo4j_util --- src-py/db_controller.py | 12 ------------ src-py/neo4j_util.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index dcc22f7c..b76de5fe 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -44,16 +44,6 @@ class DB_op(object): # for k, v in self.id_to_statement_map: # yield {k, v, None} - def __type_check_filter_attr_map(self, filter_attr_map): - """ - # type sanity check an attribute filter map - """ - assert isinstance(filter_attr_map, dict) - assert len(filter_attr_map) > 0 - for k, v in filter_attr_map.items(): - assert isinstance(k, basestring) - assert isinstance(v, list) - @property def statement_set(self): return self.id_to_statement_map.values() @@ -174,7 +164,6 @@ class DBO_load_node_set(DB_op): @return: loaded node set or an empty set if no match was found """ - self.__type_check_filter_attr_map(filter_attr_map) filter_str = dbu.where_clause_from_filter_attr_map() super(DBO_load_node_set, self).__init__() @@ -204,7 +193,6 @@ class DBO_load_link_id_set(DB_op): attributes to match link properties against @return: a set of loaded link ids """ - self.__type_check_filter_attr_map(filter_attr_map) filter_str = dbu.where_clause_from_filter_attr_map() diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index c340e15a..2d870fcc 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -69,3 +69,15 @@ def where_clause_from_filter_attr_map(filter_attr_map, node_param_name="n"): filter_str = "where {0}".format(' and '.join(filter_arr)) return filter_str +def __type_check_link(link): + assert link.has_key('__src') + assert link.has_key('__dst') + +def __type_check_filter_attr_map(filter_attr_map): + """ + # type sanity check an attribute filter map + """ + assert isinstance(filter_attr_map, dict) + for k, v in filter_attr_map.items(): + assert isinstance(k, basestring) + assert isinstance(v, list) -- cgit v1.3.1 From e69fcf28ea7fe5af9c46b71dd03e86b75f885e27 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 7 Oct 2014 18:10:29 +0200 Subject: cypher query gen utility functions: create_query_from_node_map, create_query_from_link_map --- src-py/neo4j_util.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 2d870fcc..bbd2a176 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -69,6 +69,48 @@ def where_clause_from_filter_attr_map(filter_attr_map, node_param_name="n"): filter_str = "where {0}".format(' and '.join(filter_arr)) return filter_str +def create_query_from_node_map(node_map, input_to_DB_property_map=lambda _: _): + """ + generate a set of node create queries + + @param node_map: is a node-type to node map + @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas + + @return: a (query, query_parameteres) set of create queries + """ + ret = [] + for n_type, n_set in node_map.items(): + q = "create (n:{0} {{prop_dict}}) return id(n)".format(n_type) + for n_prop_set in n_set: + q_params = {'prop_dict' : input_to_DB_property_map(n_prop_set)} + ret.append((q, q_params)) + return ret + +def create_query_from_link_map(link_map, input_to_DB_property_map=lambda _: _): + """ + generate a set of link create queries + + @param link_map: is a link-type to link map - see model.link + """ + ret = [] + for l_type, l_set in link_map.items(): + + for link in l_set: + __type_check_link(link) + + n_src = link['__src'] + n_dst = link['__dst'] + + # TODO: use object based link representation + prop_dict = link.copy() + del prop_dict['__dst'] + del prop_dict['__src'] + + q = "create ({ns})-[:{lt} {{prop_dict}}]-({nd})".format(ns=n_src, lt=l_type, nd=n_dst) + q_params = {'prop_dict' : input_to_DB_property_map(prop_dict)} + ret.append((q, q_params)) + + return ret def __type_check_link(link): assert link.has_key('__src') assert link.has_key('__dst') -- cgit v1.3.1 From 46063f7257ae06cb4a8c74f6069292aca2db2cea Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 7 Oct 2014 23:31:40 +0200 Subject: Cypher_String_Formatter - see class doc --- src-py/neo4j_util.py | 54 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 10 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index bbd2a176..78602fe1 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -4,6 +4,28 @@ import json import urllib2 +import model +import string + +class Cypher_String_Formatter(string.Formatter): + """ + Despite parameter support in Cypher, we sometimes do engage in query string building + - as both Cypher & Python use brackets to wrap parameters, escaping them in Python makes + queries less readable. This customized formatter will simply ignore unavailable keyworded + formatting arguments, allowing the use of non-escaped parameter designation, eg: + q = cfmt("match (a:{type} {cypher_param})", type='Book') + """ + + def get_field(self, field_name, args, kwargs): + # ignore key not found, return bracket wrapped key + try: + val=super(Cypher_String_Formatter, self).get_field(field_name, args, kwargs) + except (KeyError, AttributeError): + val="{" + field_name + "}", field_name + return val + +def cfmt(fmt_str, *args, **kwargs): + return Cypher_String_Formatter().format(fmt_str, *args, **kwargs) def post_neo4j(url, data): """ @@ -49,7 +71,7 @@ def statement_set_to_REST_form(statement_set): return {'statements': statement_set} -def where_clause_from_filter_attr_map(filter_attr_map, node_param_name="n"): +def gen_clause_where_from_filter_attr_map(filter_attr_map, node_label="n"): """ convert a filter attribute map to a parameterized Cypher where clause, eg. in: { 'att_foo': [ 'a', 'b' ], 'att_goo': [1,2] } @@ -59,17 +81,19 @@ def where_clause_from_filter_attr_map(filter_attr_map, node_param_name="n"): """ if not filter_attr_map: return "" - + + __type_check_filter_attr_map(filter_attr_map) + filter_arr = [] - for k in filter_attr_map.keys(): + for attr in filter_attr_map.keys(): # create a cypher query parameter place holder for each attr set # eg. n.foo in {foo}, where foo is passed as a query parameter - f_attr = "{0}.{1} in {{{1}}}".format(node_param_name, k) + f_attr = cfmt("{node_label}.{attr} in {{{attr}}}", node_label=node_label, attr=attr) filter_arr.append(f_attr) filter_str = "where {0}".format(' and '.join(filter_arr)) return filter_str -def create_query_from_node_map(node_map, input_to_DB_property_map=lambda _: _): +def gen_query_create_from_node_map(node_map, input_to_DB_property_map=lambda _: _): """ generate a set of node create queries @@ -78,15 +102,17 @@ def create_query_from_node_map(node_map, input_to_DB_property_map=lambda _: _): @return: a (query, query_parameteres) set of create queries """ + __type_check_link_or_node_map(node_map) + ret = [] for n_type, n_set in node_map.items(): - q = "create (n:{0} {{prop_dict}}) return id(n)".format(n_type) + q = cfmt("create (n:{n_type} {node_attr}) return id(n)", n_type=n_type) for n_prop_set in n_set: - q_params = {'prop_dict' : input_to_DB_property_map(n_prop_set)} + q_params = {'node_attr': input_to_DB_property_map(n_prop_set)} ret.append((q, q_params)) return ret -def create_query_from_link_map(link_map, input_to_DB_property_map=lambda _: _): +def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: _): """ generate a set of link create queries @@ -106,15 +132,23 @@ def create_query_from_link_map(link_map, input_to_DB_property_map=lambda _: _): del prop_dict['__dst'] del prop_dict['__src'] - q = "create ({ns})-[:{lt} {{prop_dict}}]-({nd})".format(ns=n_src, lt=l_type, nd=n_dst) - q_params = {'prop_dict' : input_to_DB_property_map(prop_dict)} + q = cfmt("match (src {src_attr}),(dst {dst_attr}) create (src)-[:{l_type} {link_attr}]-(dst)", l_type=l_type) + q_params = {'src_attr': { 'id': n_src} , + 'dst_attr': { 'id': n_dst} , + 'link_attr' : input_to_DB_property_map(prop_dict)} ret.append((q, q_params)) return ret + def __type_check_link(link): assert link.has_key('__src') assert link.has_key('__dst') +def __type_check_link_or_node_map(x_map): + for k, v in x_map.iteritems(): # do some type sanity checking + assert isinstance(k, basestring) + assert isinstance(v, list) + def __type_check_filter_attr_map(filter_attr_map): """ # type sanity check an attribute filter map -- cgit v1.3.1 From 3f580af48a7738fe3ff539a63918f4a22da6e91f Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 8 Oct 2014 01:35:28 +0200 Subject: gen_query_create_from_node_map: reuse single query with a multiple parameter sets --- src-py/neo4j_util.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 78602fe1..61b40c06 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -103,13 +103,15 @@ def gen_query_create_from_node_map(node_map, input_to_DB_property_map=lambda _: @return: a (query, query_parameteres) set of create queries """ __type_check_link_or_node_map(node_map) - + ret = [] for n_type, n_set in node_map.items(): q = cfmt("create (n:{n_type} {node_attr}) return id(n)", n_type=n_type) + q_params_set = [] for n_prop_set in n_set: - q_params = {'node_attr': input_to_DB_property_map(n_prop_set)} - ret.append((q, q_params)) + q_params = input_to_DB_property_map(n_prop_set) + q_params_set.append(q_params) + ret.append((q, {'node_attr': q_params_set})) return ret def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: _): @@ -118,7 +120,7 @@ def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: @param link_map: is a link-type to link map - see model.link """ - ret = [] + q_params_set = [] for l_type, l_set in link_map.items(): for link in l_set: @@ -132,13 +134,13 @@ def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: del prop_dict['__dst'] del prop_dict['__src'] - q = cfmt("match (src {src_attr}),(dst {dst_attr}) create (src)-[:{l_type} {link_attr}]-(dst)", l_type=l_type) - q_params = {'src_attr': { 'id': n_src} , - 'dst_attr': { 'id': n_dst} , + q_params = {'src': { 'id': n_src} , + 'dst': { 'id': n_dst} , 'link_attr' : input_to_DB_property_map(prop_dict)} - ret.append((q, q_params)) + q_params_set.append(q_params) - return ret + q = "match (src {id: {src}.id}),(dst {id: {dst}.id}) create (src)-[:%(l_type)s {link_attr}]->(dst)" % {'l_type':l_type} + return (q, q_params_set) def __type_check_link(link): assert link.has_key('__src') -- cgit v1.3.1 From db91e66762e1fdfc737788d8ba89bb1a0996fc5a Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 12:43:54 +0200 Subject: allow query sets to partially succeed --- src-py/neo4j_util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 61b40c06..6b78dc18 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -30,13 +30,12 @@ def cfmt(fmt_str, *args, **kwargs): def post_neo4j(url, data): """ @return dict object from the neo4j json POST response - @raise exception: if the 'errors' key is not empty """ ret = post(url, data) ret_data = json.load(ret) - if ret_data['errors']: - raise Exception('neo4j exception: ' + str(ret_data['errors'])) + # [!] do not raise exception if ret_data['errors'] is not empty - + # this allows query-sets to partially succeed return ret_data -- cgit v1.3.1 From 8e20a72f5cd0464f673a374e5a3741a58932e770 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 15:17:17 +0200 Subject: return DB_result_set when iterating over DB_op results --- src-py/db_controller.py | 4 +++- src-py/neo4j_util.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 3969a2bb..32d5a436 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -9,6 +9,7 @@ import traceback import urllib2 import neo4j_util as dbu +from neo4j_util import DB_result_set log = logging.getLogger('rhizi') @@ -50,7 +51,8 @@ class DB_op(object): i = 0 if self.result_set: for s in self.statement_set: - yield (i, s, self.result_set[i]) + rs = DB_result_set(self.result_set[i]) + yield (i, s, rs) i = i + 1 else: for s in self.statement_set: diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 6b78dc18..7c228dea 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -7,6 +7,17 @@ import urllib2 import model import string +class DB_result_set(object): + def __init__(self, data): + self.data = data + + def __iter__(self): + for db_row_dict in self.data['data']: + # example: dict: {u'row': [{u'title': u'foo'}]} + assert None != db_row_dict['row'][0] + + yield db_row_dict['row'][0] + class Cypher_String_Formatter(string.Formatter): """ Despite parameter support in Cypher, we sometimes do engage in query string building -- cgit v1.3.1 From 5f9a2481861c27f32208671f166785ba1e9cbdda Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 15 Oct 2014 18:39:58 +0200 Subject: update gen_clause_attr_filter_from_filter_attr_map() --- src-py/neo4j_util.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 7c228dea..1c4ee081 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -81,6 +81,21 @@ def statement_set_to_REST_form(statement_set): return {'statements': statement_set} +def gen_clause_attr_filter_from_filter_attr_map(filter_attr_map, node_label="n"): + if not filter_attr_map: + return "{}" + + __type_check_filter_attr_map(filter_attr_map) + + filter_arr = [] + for attr_name in filter_attr_map.keys(): + # create a cypher query parameter place holder for each attr set + # eg. n.foo in {foo}, where foo is passed as a query parameter + f_attr = cfmt("{attr_name}: {{{attr}}}", attr_name=attr_name) + filter_arr.append(f_attr) + + filter_str = "{{{0}}}".format(', '.join(filter_arr)) + return filter_str def gen_clause_where_from_filter_attr_map(filter_attr_map, node_label="n"): """ convert a filter attribute map to a parameterized Cypher where clause, eg. -- cgit v1.3.1 From 4edff7438274771d30849633b140d40a434fd491 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 15 Oct 2014 18:40:54 +0200 Subject: meta_attr_list_to_meta_attr_map - used for '__type' injected meta attributes --- src-py/neo4j_util.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 1c4ee081..6b79a84c 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -164,6 +164,28 @@ def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: 'link_attr' : input_to_DB_property_map(prop_dict)} q_params_set.append(q_params) +def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__type'): + """ + convert a list of maps each containing a meta_attr key into a + meta_attr-mapped collection of lists with the meta_attr removed - eg: + + in: [{'id':0, '__type': 'T'}, {'id':1, '__type': 'T'}] + out: { 'T', [{'id':0}, {'id':1}] } + """ + ret = {} + for v in e_set: + assert None != v['__type'] # attert type meta-attr is present + + v_type = v['__type'] + if None == ret.get(v_type):# init type list if necessary + ret[v_type] = [] + + v_no_meta = v.copy() + del v_no_meta['__type'] + + ret[v_type].append(v_no_meta) + + return ret q = "match (src {id: {src}.id}),(dst {id: {dst}.id}) create (src)-[:%(l_type)s {link_attr}]->(dst)" % {'l_type':l_type} return (q, q_params_set) -- cgit v1.3.1 From 1864fdcb50632be4882a006665f749bd2f404fff Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 15 Oct 2014 18:49:13 +0200 Subject: gen_query_create_from_link_map - revert support for parameterized link create statements --- src-py/neo4j_util.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 6b79a84c..120a601a 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -145,8 +145,14 @@ def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: @param link_map: is a link-type to link map - see model.link """ - q_params_set = [] + __type_check_link_or_node_map(link_map) + + ret = [] for l_type, l_set in link_map.items(): + q = "match (src {id: {src}.id}),(dst {id: {dst}.id}) " + \ + "create (src)-[r:%(__type)s {link_attr}]->(dst) " + \ + "return id(r)" + q = q % {'__type':l_type} for link in l_set: __type_check_link(link) @@ -162,7 +168,9 @@ def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: q_params = {'src': { 'id': n_src} , 'dst': { 'id': n_dst} , 'link_attr' : input_to_DB_property_map(prop_dict)} - q_params_set.append(q_params) + ret.append((q, q_params)) + + return ret def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__type'): """ -- cgit v1.3.1 From 12b475ab0f131c6db8f5b28b7d5e5a89d11be9cf Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 15 Oct 2014 18:54:36 +0200 Subject: misc --- src-py/db_controller.py | 40 ++++++++++++++++++++-------------------- src-py/neo4j_util.py | 19 ++++++++++++------- 2 files changed, 32 insertions(+), 27 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index dcb70c10..cb52cc09 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -156,13 +156,10 @@ class DBO_add_link_set(DB_op): self.add_statement(q, q_params) def on_completion(self, data): - super(DBO_add_link_set, self).on_completion(data) - id_set = [] for s_id, s, r_set in self: for row in r_set: - # [!] fragile - parse results - lid = row + lid = row # [!] fragile id_set.append(lid) log.debug('link-set added: ids: ' + str(id_set)) @@ -183,7 +180,7 @@ class DBO_load_node_set_by_DB_id(DB_op): log.debug('loaded node set: ' + str(data)) return self.parse_single_query_response_data(data) -class DBO_load_node_set(DB_op): +class DBO_match_node_id_set(DB_op): def __init__(self, filter_type=None, filter_attr_map=None): """ @@ -208,24 +205,26 @@ class DBO_load_node_set(DB_op): log.debug('loaded id-set: ' + str(data)) return self.parse_single_query_response_data(data) -class DBO_load_node_set_by_id_attribute(DBO_load_node_id_set): +class DBO_match_node_set_by_id_attribute(DBO_match_node_id_set): def __init__(self, id_set): """ convenience op: load a set of nodes by their 'id' attribute != DB node id """ assert isinstance(id_set, list) - super(DBO_load_node_set_by_id_attribute, self).__init__(filter_attr_map={'id': id_set}) + super(DBO_match_node_set_by_id_attribute, self).__init__(filter_attr_map={'id': id_set}) -class DBO_load_link_set_by_src_or_dst_id_attributes(DB_op): +class DBO_match_link_set_by_src_or_dst_id_attributes(DB_op): def __init__(self, src_id=None, dst_id=None): """ - load an id-set of links by source/target id attributes != DB node id + match a set of links by source/target node id attributes + + @return: a set of loaded links """ assert None != src_id or None != dst_id - super(DBO_load_link_set_by_src_or_dst_id_attributes, self).__init__() + super(DBO_match_link_set_by_src_or_dst_id_attributes, self).__init__() if not src_id: q = "match ()-[r]->({id: {dst_id}}) return r" @@ -243,7 +242,7 @@ class DBO_load_link_set_by_src_or_dst_id_attributes(DB_op): log.debug('loaded id-set: ' + str(data)) return self.parse_single_query_response_data(data) -class DBO_load_link_id_set(DB_op): +class DBO_match_link_id_set(DB_op): def __init__(self, filter_type=None, filter_attr_map={}): """ load an id-set of links @@ -253,7 +252,7 @@ class DBO_load_link_id_set(DB_op): attributes to match link properties against @return: a set of loaded link ids """ - super(DBO_load_link_id_set, self).__init__() + super(DBO_match_link_id_set, self).__init__() q = "match ()-[r{filter_type} {filter_attr}]->() return id(r)" q = cfmt(q, filter_type="" if not filter_type else ":" + filter_type) @@ -280,8 +279,8 @@ class DB_Driver_REST(DB_Driver_Base): # # [!] neo4j seems picky about receiving an additional empty statement list # - data = data = dbu.statement_set_to_REST_form([]) - ret = dbu.post_neo4j(tx_open_url, data) + data = data = db_util.statement_set_to_REST_form([]) + ret = db_util.post_neo4j(tx_open_url, data) tx_commit_url = ret['commit'] op.parse_tx_id(tx_commit_url) @@ -291,10 +290,11 @@ class DB_Driver_REST(DB_Driver_Base): def exex_op_statements(self, op): tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) - statement_set = dbu.statement_set_to_REST_form(op.statement_set) + statement_set = db_util.statement_set_to_REST_form(op.statement_set) try: - ret = dbu.post_neo4j(tx_url, statement_set) + ret = db_util.post_neo4j(tx_url, statement_set) + op._assign_results_errors(ret) self.log_committed_queries(statement_set) return ret except Exception as e: @@ -307,8 +307,8 @@ class DB_Driver_REST(DB_Driver_Base): # # [!] neo4j seems picky about receiving an additional empty statement list # - data = dbu.statement_set_to_REST_form([]) - ret = dbu.post(tx_commit_url, data) + data = db_util.statement_set_to_REST_form([]) + ret = db_util.post(tx_commit_url, data) log.debug('tx-commit: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) @@ -379,5 +379,5 @@ class DB_Controller: @deprecated: use transaction based api """ - # call post and not dbu.post_neo4j to avoid response key errors - dbu.post(self.config.db_base_url + '/db/data/cypher', {"query" : q}) + # call post and not db_util.post_neo4j to avoid response key errors + db_util.post(self.config.db_base_url + '/db/data/cypher', {"query" : q}) diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 120a601a..5169af0d 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -30,9 +30,9 @@ class Cypher_String_Formatter(string.Formatter): def get_field(self, field_name, args, kwargs): # ignore key not found, return bracket wrapped key try: - val=super(Cypher_String_Formatter, self).get_field(field_name, args, kwargs) + val = super(Cypher_String_Formatter, self).get_field(field_name, args, kwargs) except (KeyError, AttributeError): - val="{" + field_name + "}", field_name + val = "{" + field_name + "}", field_name return val def cfmt(fmt_str, *args, **kwargs): @@ -72,7 +72,11 @@ def statement_to_REST_form(query, parameters={}): turn cypher query to neo4j json API format """ assert isinstance(query, basestring) - assert isinstance(parameters, dict) + if isinstance(parameters, list): + for v in parameters: + assert isinstance(v, dict) + else: + assert isinstance(parameters, dict) return {'statement' : query, 'parameters': parameters} @@ -96,13 +100,16 @@ def gen_clause_attr_filter_from_filter_attr_map(filter_attr_map, node_label="n") filter_str = "{{{0}}}".format(', '.join(filter_arr)) return filter_str + def gen_clause_where_from_filter_attr_map(filter_attr_map, node_label="n"): """ convert a filter attribute map to a parameterized Cypher where clause, eg. in: { 'att_foo': [ 'a', 'b' ], 'att_goo': [1,2] } - out: where n.att_foo in {att_foo} and n.att_goo in {att_goo} ... + out: {att_foo: {att_foo}, att_goo: {att_goo}, ...} + + this function will essentially ignore all but the first value in the value list - @param filter_attr_map: may be None or empty + @param filter_attr_map: may be None or empty """ if not filter_attr_map: return "" @@ -194,8 +201,6 @@ def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__type'): ret[v_type].append(v_no_meta) return ret - q = "match (src {id: {src}.id}),(dst {id: {dst}.id}) create (src)-[:%(l_type)s {link_attr}]->(dst)" % {'l_type':l_type} - return (q, q_params_set) def __type_check_link(link): assert link.has_key('__src') -- cgit v1.3.1 From d3d8bd18bfbf2da267ea30f5625f092d24b21474 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Thu, 16 Oct 2014 16:01:33 +0200 Subject: DB_row, update DBO_XXX iteration loops, test updates --- src-py/db_controller.py | 15 +++++++++------ src-py/neo4j_util.py | 12 ++++++++++-- src-py/test_db_controller.py | 14 ++++++++++---- 3 files changed, 29 insertions(+), 12 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 49ed3b6b..f4bfbb75 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -67,7 +67,8 @@ class DB_op(object): ret = [] for _, _, r_set in self: for row in r_set: - ret.append(row) + for cloumn in row: + ret.append(row) return ret def parse_multi_statement_response_data(self, data): @@ -155,6 +156,7 @@ class DBO_add_node_set(DB_op): DB op: add node set @param node_map: node-type to node-set map + @return: set of new node DB ids """ super(DBO_add_node_set, self).__init__() for q, q_param_set in db_util.gen_query_create_from_node_map(node_map): @@ -164,8 +166,8 @@ class DBO_add_node_set(DB_op): id_set = [] for _, _, r_set in self: for row in r_set: - nid = row # [!] fragile - id_set.append(nid) + for clo in row: + id_set.append(clo) log.debug('node-set added: ids: ' + str(id_set)) return id_set @@ -174,6 +176,7 @@ class DBO_add_link_set(DB_op): def __init__(self, link_map): """ @param link_map: is a link-type to link-set map - see model.link + @return: set of new node DB ids """ super(DBO_add_link_set, self).__init__() for q, q_params in db_util.gen_query_create_from_link_map(link_map): @@ -181,10 +184,10 @@ class DBO_add_link_set(DB_op): def on_completion(self, data): id_set = [] - for s_id, s, r_set in self: + for _, _, r_set in self: for row in r_set: - lid = row # [!] fragile - id_set.append(lid) + for col_val in row: + id_set.append(col_val) log.debug('link-set added: ids: ' + str(id_set)) return id_set diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 5169af0d..8d70ced9 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -7,6 +7,14 @@ import urllib2 import model import string +class DB_row(object): + def __init__(self, data): + self.data = data + + def __iter__(self): + for column_val in self.data: + yield column_val + class DB_result_set(object): def __init__(self, data): self.data = data @@ -14,9 +22,9 @@ class DB_result_set(object): def __iter__(self): for db_row_dict in self.data['data']: # example: dict: {u'row': [{u'title': u'foo'}]} - assert None != db_row_dict['row'][0] + assert None != db_row_dict['row'] - yield db_row_dict['row'][0] + yield DB_row(db_row_dict['row']) class Cypher_String_Formatter(string.Formatter): """ diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 66ecfd09..edfd4a22 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -135,13 +135,19 @@ class TestDBController(unittest.TestCase): n_set = self.db_ctl.exec_op(op) self.assertEqual(len(n_set), 1) - def test_node_DB_id_lifecycle(self): + def test_load_node_set_by_DB_id(self): """ test node DB id life cycle """ - id_set = self.db_ctl.exec_op(dbc.DBO_add_node_set({'Person': [{'name': 'John Doe', 'id': 'jdoe_00'}, - {'name': 'John Doe', 'id': 'jdoe_01'}]})) - n_set = self.db_ctl.exec_op(dbc.DBO_load_node_set_by_DB_id(id_set)) + + # create nodes, get DB ids + op = dbc.DBO_add_node_set({'T_test_load_node_set_by_DB_id': [{'name': 'John Doe'}, + {'name': 'John Doe'}]}) + id_set = self.db_ctl.exec_op(op) + + # match against DB ids + op = dbc.DBO_load_node_set_by_DB_id(id_set) + n_set = self.db_ctl.exec_op(op) self.assertEqual(len(n_set), len(id_set), 'incorrect result size') def test_partial_query_set_execution_success(self): -- cgit v1.3.1 From 957076abcebb5dc898852b6c58733fa0b408b008 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 20 Oct 2014 19:31:06 +0200 Subject: Neo4JException --- src-py/db_controller.py | 14 ++++++++++---- src-py/neo4j_util.py | 7 +++++++ 2 files changed, 17 insertions(+), 4 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 875eb0bc..7cd1af7b 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -351,15 +351,21 @@ class DB_Driver_REST(DB_Driver_Base): except Exception as e: raise Exception('failed to open transaction:' + e.message) - def exex_op_statements(self, op): + def exec_statement_set(self, op): + tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) statement_set = db_util.statement_set_to_REST_form(op.statement_set) try: - ret = db_util.post_neo4j(tx_url, statement_set) - op._assign_results_errors(ret) + post_ret = db_util.post_neo4j(tx_url, statement_set) + op.result_set = post_ret['results'] + op.error_set = post_ret['errors'] + if 0 != len(op.error_set): + raise Neo4JException(op.error_set) + self.log_committed_queries(statement_set) - return ret + except Neo4JException as e: + raise e except Exception as e: raise Exception('failed exec op statements: err: {0}, url: {1}'.format(e.message, tx_url)) diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 8d70ced9..5b7cfee7 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -7,6 +7,13 @@ import urllib2 import model import string +class Neo4JException(Exception): + def __init__(self, error_set): + self.error_set = error_set + + def __str__(self): + return 'neo4j error set: ' + str(self.error_set) + class DB_row(object): def __init__(self, data): self.data = data -- cgit v1.3.1 From d0278d6627d5b1d50ac0697307a3259f469698f8 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 21 Oct 2014 19:58:19 +0200 Subject: enable neo4j JSON streaming by default --- src-py/neo4j_util.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 5b7cfee7..642e094a 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -75,6 +75,8 @@ def post(url, data): req.add_header('Accept', 'application/json; charset=UTF-8') req.add_header('Content-Type', 'application/json') + req.add_header('X-Stream', 'true') # enable neo4j JSON streaming + try: ret = urllib2.urlopen(req, post_data_json) except urllib2.HTTPError as e: -- cgit v1.3.1 From 48e483a6f7bb11c22b810341088f441706474a06 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 9 Nov 2014 12:12:57 +0200 Subject: misc --- src-py/db_controller.py | 4 ++++ src-py/neo4j_util.py | 19 ++++++++++++++----- src-py/test_db_controller.py | 29 ++++++++++++++++++----------- 3 files changed, 36 insertions(+), 16 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 273686fd..d9d8d737 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -63,6 +63,10 @@ class DB_op(object): def parse_multi_statement_response_data(self, data): pass + @property + def name(self): + return self.__class__.__name__ + def process_result_set(self): """ DB op can issue complex sets of quries all at once - this helper method diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 642e094a..2b6257b4 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -6,14 +6,17 @@ import json import urllib2 import model import string +import time + +from util import debug_log_duration class Neo4JException(Exception): def __init__(self, error_set): self.error_set = error_set - + def __str__(self): return 'neo4j error set: ' + str(self.error_set) - + class DB_row(object): def __init__(self, data): self.data = data @@ -22,6 +25,9 @@ class DB_row(object): for column_val in self.data: yield column_val + def items(self): + return [x for x in self] + class DB_result_set(object): def __init__(self, data): self.data = data @@ -33,6 +39,9 @@ class DB_result_set(object): yield DB_row(db_row_dict['row']) + def items(self): + return [x for x in self] + class Cypher_String_Formatter(string.Formatter): """ Despite parameter support in Cypher, we sometimes do engage in query string building @@ -206,12 +215,12 @@ def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__type'): """ ret = {} for v in e_set: - assert None != v['__type'] # attert type meta-attr is present + assert None != v['__type'] # assert type meta-attr is present v_type = v['__type'] - if None == ret.get(v_type):# init type list if necessary + if None == ret.get(v_type): # init type list if necessary ret[v_type] = [] - + v_no_meta = v.copy() del v_no_meta['__type'] diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 4ff0c4de..da8ae364 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -5,6 +5,7 @@ import db_controller as dbc from rhizi_server import Config from neo4j_test_util import rand_id from neo4j_test_util import flush_db +from neo4j_test_util import gen_rand_data from neo4j_util import Neo4JException from model.graph import Attr_Diff @@ -35,8 +36,12 @@ class TestDBController(unittest.TestCase): self.log = logging.getLogger('rhizi') self.log.addHandler(logging.StreamHandler()) + # TODO rm when implemented: neo4j_test_util + self.db_ctl.exec_cypher_query('create index on :Person(id)') + self.db_ctl.exec_cypher_query('create index on :Skill(id)') + def setUp(self): - flush_db(self.db_ctl) # remove once embedded DB test mode is supported + flush_db(self.db_ctl) # remove once embedded DB test mode is supported self.db_ctl.exec_op(dbc.DBO_add_node_set(self.n_map)) self.db_ctl.exec_op(dbc.DBO_add_link_set(self.l_map)) @@ -127,6 +132,8 @@ class TestDBController(unittest.TestCase): self.assertEqual(len(id_set), 0) def test_load_link_set(self): + + # load by l_ptr l_ptr = Link.link_ptr(src_id='person_00', dst_id='skill_00') op = dbc.DBO_load_link_set.init_from_link_ptr(l_ptr) l_set = self.db_ctl.exec_op(op) @@ -164,7 +171,7 @@ class TestDBController(unittest.TestCase): op = dbc.DBO_add_node_set({'T_test_load_node_set_by_DB_id': [{'name': 'John Doe'}, {'name': 'John Doe'}]}) id_set = self.db_ctl.exec_op(op) - + # match against DB ids op = dbc.DBO_load_node_set_by_DB_id(id_set) n_set = self.db_ctl.exec_op(op) @@ -219,16 +226,16 @@ class TestDBController(unittest.TestCase): id_set = self.db_ctl.exec_op(dbc.DBO_match_node_set_by_id_attribute([n_0_id, n_1_id])) self.assertEqual(len(id_set), 2) - + l_ptr = Link.link_ptr(src_id=n_0_id, dst_id=n_1_id) id_set = self.db_ctl.exec_op(dbc.DBO_load_link_set.init_from_link_ptr(l_ptr)) self.assertEqual(len(id_set), 1) - + l_ptr = Link.link_ptr(src_id=n_1_id, dst_id=n_0_id) id_set = self.db_ctl.exec_op(dbc.DBO_load_link_set.init_from_link_ptr(l_ptr)) self.assertEqual(len(id_set), 1) - id_set_rm=[n_2_id] + id_set_rm = [n_2_id] topo_diff = Topo_Diff(node_set_rm=id_set_rm) op = dbc.DBO_topo_diff_commit(topo_diff) self.db_ctl.exec_op(op) @@ -278,7 +285,7 @@ class TestDBController(unittest.TestCase): n_2_id = rand_id() n_3_id = rand_id() n_T = 'T_test_rm_node_set' - + n_set = [{'__type': n_T, 'id': n_0_id }, {'__type': n_T, 'id': n_1_id }, {'__type': n_T, 'id': n_2_id }, @@ -291,13 +298,13 @@ class TestDBController(unittest.TestCase): op = dbc.DBO_topo_diff_commit(topo_diff) self.db_ctl.exec_op(op) - + op = dbc.DBO_rm_node_set([n_0_id, n_1_id]) self.db_ctl.exec_op(op) - + op = dbc.DBO_rm_node_set([n_2_id, n_3_id], rm_links=True) self.db_ctl.exec_op(op) - + # assert all deleted op = dbc.DBO_match_node_id_set(filter_type=n_T) id_set = self.db_ctl.exec_op(op) @@ -305,11 +312,11 @@ class TestDBController(unittest.TestCase): def test_rz_clone(self): l_n, l_r = gen_rand_data(self.db_ctl, lim_n=8, lim_r=16, prob_link_create=0.7) - op = dbc.DBO_rz_clone(filter_label = l_n, limit=32) + op = dbc.DBO_rz_clone(filter_label=l_n, limit=32) ret = self.db_ctl.exec_op(op) n_set = ret['node_set'] l_set = ret['link_set'] - + # TODO improve assertions self.assertTrue(0 < len(n_set)) self.assertTrue(0 < len(l_set)) -- cgit v1.3.1 From 12506bd331378a2582ff1e54f1b4e09910b7342d Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Mon, 24 Nov 2014 12:27:16 +0200 Subject: python3 compat: urllib --- src-py/neo4j_util.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 2b6257b4..1d0f0203 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -3,7 +3,8 @@ """ import json -import urllib2 +from six.moves.urllib import request +import six.moves.urllib_error as urllib_error import model import string import time @@ -79,7 +80,7 @@ def post(url, data): post_data_json = json.dumps(data) - req = urllib2.Request(url) + req = request.Request(url) req.add_header('User-Agent', 'rhizi-server/0.1') req.add_header('Accept', 'application/json; charset=UTF-8') req.add_header('Content-Type', 'application/json') @@ -87,8 +88,8 @@ def post(url, data): req.add_header('X-Stream', 'true') # enable neo4j JSON streaming try: - ret = urllib2.urlopen(req, post_data_json) - except urllib2.HTTPError as e: + ret = request.urlopen(req, post_data_json) + except urllib_error.HTTPError as e: raise Exception('post request failed: code: {0}, reason: {1}'.format(e.code, e.reason)) return ret -- cgit v1.3.1 From 83e348827e685798b2200252698c6705e57f7647 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Mon, 24 Nov 2014 12:27:42 +0200 Subject: neo4j_util: whitespace before EOL fixes --- src-py/neo4j_util.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 1d0f0203..ff0fca96 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -45,9 +45,9 @@ class DB_result_set(object): class Cypher_String_Formatter(string.Formatter): """ - Despite parameter support in Cypher, we sometimes do engage in query string building + Despite parameter support in Cypher, we sometimes do engage in query string building - as both Cypher & Python use brackets to wrap parameters, escaping them in Python makes - queries less readable. This customized formatter will simply ignore unavailable keyworded + queries less readable. This customized formatter will simply ignore unavailable keyworded formatting arguments, allowing the use of non-escaped parameter designation, eg: q = cfmt("match (a:{type} {cypher_param})", type='Book') """ @@ -133,9 +133,9 @@ def gen_clause_where_from_filter_attr_map(filter_attr_map, node_label="n"): convert a filter attribute map to a parameterized Cypher where clause, eg. in: { 'att_foo': [ 'a', 'b' ], 'att_goo': [1,2] } out: {att_foo: {att_foo}, att_goo: {att_goo}, ...} - - this function will essentially ignore all but the first value in the value list - + + this function will essentially ignore all but the first value in the value list + @param filter_attr_map: may be None or empty """ if not filter_attr_map: @@ -155,10 +155,10 @@ def gen_clause_where_from_filter_attr_map(filter_attr_map, node_label="n"): def gen_query_create_from_node_map(node_map, input_to_DB_property_map=lambda _: _): """ generate a set of node create queries - + @param node_map: is a node-type to node map @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas - + @return: a (query, query_parameteres) set of create queries """ __type_check_link_or_node_map(node_map) @@ -176,7 +176,7 @@ def gen_query_create_from_node_map(node_map, input_to_DB_property_map=lambda _: def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: _): """ generate a set of link create queries - + @param link_map: is a link-type to link map - see model.link """ __type_check_link_or_node_map(link_map) @@ -210,7 +210,7 @@ def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__type'): """ convert a list of maps each containing a meta_attr key into a meta_attr-mapped collection of lists with the meta_attr removed - eg: - + in: [{'id':0, '__type': 'T'}, {'id':1, '__type': 'T'}] out: { 'T', [{'id':0}, {'id':1}] } """ -- cgit v1.3.1 From 89f4e817f13b4359ad8c3ca8fa37fdece2b3aa1d Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 8 Dec 2014 22:35:07 +0200 Subject: python 3: switch from basestring to six.string_types --- src-py/neo4j_util.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index ff0fca96..6c46008c 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -3,6 +3,7 @@ """ import json +import six from six.moves.urllib import request import six.moves.urllib_error as urllib_error import model @@ -98,7 +99,7 @@ def statement_to_REST_form(query, parameters={}): """ turn cypher query to neo4j json API format """ - assert isinstance(query, basestring) + assert isinstance(query, six.string_types) if isinstance(parameters, list): for v in parameters: assert isinstance(v, dict) @@ -235,7 +236,7 @@ def __type_check_link(link): def __type_check_link_or_node_map(x_map): for k, v in x_map.iteritems(): # do some type sanity checking - assert isinstance(k, basestring) + assert isinstance(k, six.string_types) assert isinstance(v, list) def __type_check_filter_attr_map(filter_attr_map): @@ -244,5 +245,5 @@ def __type_check_filter_attr_map(filter_attr_map): """ assert isinstance(filter_attr_map, dict) for k, v in filter_attr_map.items(): - assert isinstance(k, basestring) + assert isinstance(k, six.string_types) assert isinstance(v, list) -- cgit v1.3.1 From 5dc9b4a0f4c6e36fbb72b5f3365ff9100cb1359f Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 8 Dec 2014 22:36:28 +0200 Subject: use '__label_set' as default type meta_attr (only single label support for now) --- src-py/neo4j_util.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 6c46008c..d9fb1c67 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -207,7 +207,7 @@ def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: return ret -def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__type'): +def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__label_set'): """ convert a list of maps each containing a meta_attr key into a meta_attr-mapped collection of lists with the meta_attr removed - eg: @@ -217,14 +217,15 @@ def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__type'): """ ret = {} for v in e_set: - assert None != v['__type'] # assert type meta-attr is present + assert None != v[meta_attr], 'missing type meta-attribute' + assert 1 == len(v[meta_attr]), 'only single-type mapping currently suppoerted' - v_type = v['__type'] + v_type = v[meta_attr][0] if None == ret.get(v_type): # init type list if necessary ret[v_type] = [] v_no_meta = v.copy() - del v_no_meta['__type'] + del v_no_meta[meta_attr] ret[v_type].append(v_no_meta) -- cgit v1.3.1 From adaa8509398f18ce91db74d0a78f898bf9ae1a0d Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 8 Dec 2014 22:38:36 +0200 Subject: gen_query_create_from_node_map(): return n.id, assert id present --- src-py/neo4j_util.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index d9fb1c67..cc3d53d8 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -86,7 +86,7 @@ def post(url, data): req.add_header('Accept', 'application/json; charset=UTF-8') req.add_header('Content-Type', 'application/json') - req.add_header('X-Stream', 'true') # enable neo4j JSON streaming + req.add_header('X-Stream', 'true') # enable neo4j JSON streaming try: ret = request.urlopen(req, post_data_json) @@ -166,9 +166,16 @@ def gen_query_create_from_node_map(node_map, input_to_DB_property_map=lambda _: ret = [] for n_type, n_set in node_map.items(): - q = cfmt("create (n:{n_type} {node_attr}) return id(n)", n_type=n_type) + q_arr = ['create (n:%s {node_attr})' % (n_type), + 'return n.id' + ] + + q = ' '.join(q_arr) q_params_set = [] for n_prop_set in n_set: + + assert None != n_prop_set['id'], 'node create query: node id attribute not set' + q_params = input_to_DB_property_map(n_prop_set) q_params_set.append(q_params) ret.append((q, {'node_attr': q_params_set})) -- cgit v1.3.1 From 0950293605eecff2de9cdadccf1def2d7d9b73bc Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 10 Dec 2014 15:16:06 +0200 Subject: distinguish __src (obj) vs. __src_id (str) --- src-py/db_controller.py | 4 ++-- src-py/model/model.py | 25 ++++++++++++++++--------- src-py/neo4j_util.py | 16 ++++++++-------- src-py/rhizi_api.py | 4 ++-- 4 files changed, 28 insertions(+), 21 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 460686a5..2d4fe8ce 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -426,8 +426,8 @@ class DBO_rz_clone(DB_op): continue l = l_tuple[1] - l['__src'] = n['id'] - l['__dst'] = l_tuple[0] + l['__src_id'] = n['id'] + l['__dst_id'] = l_tuple[0] l['__label_set'] = [l_tuple[2]] # box single value returned by type() ret_l_set.append(l) diff --git a/src-py/model/model.py b/src-py/model/model.py index 701074ab..33cf9e6a 100644 --- a/src-py/model/model.py +++ b/src-py/model/model.py @@ -3,24 +3,31 @@ class Link(): documentation anchor - this class currently carries no implementation and only acts as a documentation anchor - link['__src'] - meta attribute for link source - link['__dst'] - meta attribute for link destination + link['__src'] - meta attribute for link source object + link['__dst'] - meta attribute for link destination object """ - + + def __init__(self, src=None, dst=None): + assert False, 'currently unused' + class Link_Ptr(dict): + """ + link['__src_id'] - meta attribute for link source id + link['__dst_id'] - meta attribute for link destination id + """ def __init__(self, src_id=None, dst_id=None): assert None != src_id or None != dst_id - - self['__src'] = src_id - self['__dst'] = dst_id - + + self['__src_id'] = src_id + self['__dst_id'] = dst_id + @property def src_id(self): - return self['__src'] + return self['__src_id'] @property def dst_id(self): - return self['__dst'] + return self['__dst_id'] @staticmethod def link_ptr(src_id=None, dst_id=None): diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index cc3d53d8..a203207f 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -199,16 +199,16 @@ def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: for link in l_set: __type_check_link(link) - n_src = link['__src'] - n_dst = link['__dst'] + src_id = link['__src_id'] + dst_id = link['__dst_id'] # TODO: use object based link representation prop_dict = link.copy() - del prop_dict['__dst'] - del prop_dict['__src'] + del prop_dict['__dst_id'] + del prop_dict['__src_id'] - q_params = {'src': { 'id': n_src} , - 'dst': { 'id': n_dst} , + q_params = {'src': { 'id': src_id} , + 'dst': { 'id': dst_id} , 'link_attr' : input_to_DB_property_map(prop_dict)} ret.append((q, q_params)) @@ -239,8 +239,8 @@ def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__label_set'): return ret def __type_check_link(link): - assert link.has_key('__src') - assert link.has_key('__dst') + assert link.has_key('__src_id') + assert link.has_key('__dst_id') def __type_check_link_or_node_map(x_map): for k, v in x_map.iteritems(): # do some type sanity checking diff --git a/src-py/rhizi_api.py b/src-py/rhizi_api.py index 8d787a5c..ef904a37 100644 --- a/src-py/rhizi_api.py +++ b/src-py/rhizi_api.py @@ -127,8 +127,8 @@ def load_link_set_by_link_ptr_set(): l_ptr_set = [] for lptr_dict in l_ptr_set_raw: - src_id = lptr_dict.get('__src') - dst_id = lptr_dict.get('__dst') + src_id = lptr_dict.get('__src_id') + dst_id = lptr_dict.get('__dst_id') l_ptr_set += [Link.Link_Ptr(src_id=src_id, dst_id=dst_id) ] return l_ptr_set -- cgit v1.3.1 From 2ce705db9086188897df8464ac9cb8cb6fb6ce0e Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 15 Dec 2014 23:38:05 +0200 Subject: gen_query_create_from_node_map(): validate label --- src-py/neo4j_util.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src-py/neo4j_util.py') diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index a203207f..cf2f962e 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -165,8 +165,11 @@ def gen_query_create_from_node_map(node_map, input_to_DB_property_map=lambda _: __type_check_link_or_node_map(node_map) ret = [] - for n_type, n_set in node_map.items(): - q_arr = ['create (n:%s {node_attr})' % (n_type), + for label, n_set in node_map.items(): + + assert len(label) > 2 and label[0].isupper() and label[1:].islower(), 'malformed label: ' + label + + q_arr = ['create (n:%s {node_attr})' % (label), 'return n.id' ] -- cgit v1.3.1