From 68b2334cd8484ac72814060ced5703a2f94b57d7 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 23 Sep 2014 16:28:16 +0300 Subject: initial implementation --- src-py/db_controller.py | 179 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 src-py/db_controller.py (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py new file mode 100644 index 00000000..6ad77b01 --- /dev/null +++ b/src-py/db_controller.py @@ -0,0 +1,179 @@ +#!/usr/bin/python + +import os +import json +import re +import logging +import traceback + +import urllib2 + +import neo4j_util as dbu + +log = logging.getLogger('rhizi') + +class DB_op(object): + """ + tx wrapped DB operation possibly composing multiple DB queries + """ + + def __init__(self): + self.s_id = 0 # statement id counter + self.id_to_statement_map = {} # zero based id to statement map + self.tx_id = None + self.tx_commit_url = None # cached from response to tx begin + + def begin(self, tx_open_url): + try: + # + # [!] neo4j seems picky about receiving an additional empty statement list + # + data = data = dbu.statement_set_to_REST_form([]) + ret = dbu.post_neo4j(tx_open_url, data) + tx_commit_url = ret['commit'] + self.parse_tx_id(tx_commit_url) + self.tx_commit_url = tx_commit_url + + log.debug('tx-open: id: {0}, commit-url: {1}'.format(self.tx_id, tx_commit_url)) + except Exception as e: + raise Exception('failed to open transaction:' + e.message) + + def parse_tx_id(self, tx_commit_url): + m = re.search('/(?P\d+)/commit$', tx_commit_url) + id_str = m.group('id') + self.tx_id = int(id_str) + + def commit(self): + try: + # + # [!] neo4j seems picky about receiving an additional empty statement list + # + data = dbu.statement_set_to_REST_form([]) + ret = dbu.post(self.tx_commit_url, data) + except Exception as e: + raise Exception('failed to commit transaction:' + e.message) + + log.debug('tx-commit: id: {0}, commit-url: {1}'.format(self.tx_id, self.tx_commit_url)) + + def add_statement(self, cypher_query, params={}): + """ + add a DB query language statement + @return: statement id + """ + ret = self.s_id + self.id_to_statement_map[self.s_id] = dbu.statement_to_REST_form(cypher_query, params) + self.s_id = self.s_id + 1 + return ret + + @property + def statement_set(self): + return self.id_to_statement_map.values() + + def on_success(self, data): + pass + + def on_error(self): + pass + +class DBO_add_node_set(DB_op): + """ + DB op: add node set + + @param node_map: type to node list map + """ + def __init__(self, node_map): + super(DBO_add_node_set, self).__init__() + self.node_map = node_map + + for type, n_set in self.node_map.items(): + q = "create (n:{0} {{prop_dict}}) return id(n)".format(type) + for n in n_set: + # + # any translation between how we accept node data + # and how we store them should go here + # + p = {'prop_dict' : { 'name' : n['name']}} + self.add_statement(q, p) + + def on_success(self, data): + # [!] fragile - parse results + # sample input: dict: {u'errors': [], u'results': [{u'data': [{u'row': [20]}], u'columns': [u'id(n)']}]} + id_set = [] + for r in data['results']: + columns = r['columns'] + for k in r['data']: + nid = k['row'][0] + id_set.append(nid) + + log.debug('node-set added: ids: ' + str(id_set)) + return id_set + +class DBO_load_node_id_set(DB_op): + """ + load node id set, filter by type / properties + """ + def __init__(self, filter_type, filter_prop=None): + super(DBO_load_node_id_set, self).__init__() + + # build where clause if necessary + filter_prop_str = "" + if filter_prop: + filter_prop_arr = [] + for k, v in filter_prop: + v_str = str(v) + if isinstance(v, str): + # quote string values + v_str = "'{0}'".format(v_str) + filter_prop_arr.append("n.{0} = {1} and ".format(k, v_str)) + filter_prop_str = " where " + " and ".join(filter_prop_arr) + + q = "match (n:{0}){1} return id(n)".format(filter_type, filter_prop_str) + self.add_statement(q) + + def on_success(self, data): + # [!] fragile - parse results + # sample input: dict: {u'errors': [], u'results': [{u'data': [{u'row': [20]}], u'columns': [u'id(n)']}]} + id_set = [] + for r in data['results']: + columns = r['columns'] + for k in r['data']: + nid = k['row'][0] + id_set.append(nid) + + log.debug('loaded node-set: ids: ' + str(id_set)) + return id_set + +class DB_Controller: + """ + neo4j DB controller + """ + def __init__(self, config): + self.config = config + + def exec_op(self, op): + """ + execute operation within a DB transaction + """ + tx_base_url = self.config.db_base_url + '/db/data/transaction' + data = dbu.statement_set_to_REST_form(op.statement_set) + + try: + op.begin(tx_base_url) + tx_url = "{0}/{1}".format(tx_base_url, op.tx_id) + ret_tx = dbu.post_neo4j(tx_url, data) + op.commit() + return op.on_success(ret_tx) + except Exception as e: + log.error(e.message) + log.error(traceback.print_exc()) + op.on_error() + + def create_db_op(self, f_work, f_cont): + ret = DB_op(f_work, f_cont) + return ret + + def exec_cypher_query(self, q): + """ + @deprecated: use transaction based api + """ + self.post_neo4j('/db/data/cypher', {"query" : q}) -- cgit v1.3.1 From 391d02c6770fd9b7dea5014bfa974f675388c5c6 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 24 Sep 2014 02:00:18 +0300 Subject: db_controller.DBO_add_node_set.__init__ - add optional input_to_DB_property_map property mapping argument --- src-py/db_controller.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 6ad77b01..d7492e95 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -79,20 +79,18 @@ class DBO_add_node_set(DB_op): """ DB op: add node set - @param node_map: type to node list map + @param node_map: node-type to node list map + @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas + """ - def __init__(self, node_map): + def __init__(self, node_map, input_to_DB_property_map=lambda _: _): super(DBO_add_node_set, self).__init__() self.node_map = node_map for type, n_set in self.node_map.items(): q = "create (n:{0} {{prop_dict}}) return id(n)".format(type) - for n in n_set: - # - # any translation between how we accept node data - # and how we store them should go here - # - p = {'prop_dict' : { 'name' : n['name']}} + for n_prop_dict in n_set: + p = {'prop_dict' : input_to_DB_property_map(n_prop_dict)} self.add_statement(q, p) def on_success(self, data): @@ -140,7 +138,7 @@ class DBO_load_node_id_set(DB_op): nid = k['row'][0] id_set.append(nid) - log.debug('loaded node-set: ids: ' + str(id_set)) + log.debug('loaded node id set: ' + str(id_set)) return id_set class DB_Controller: @@ -150,18 +148,23 @@ class DB_Controller: def __init__(self, config): self.config = config + def log_committed_queries(self, statement_set): + for sp_dict in statement_set['statements']: + log.debug('\tq: {0}'.format(sp_dict['statement'])) + def exec_op(self, op): """ execute operation within a DB transaction """ tx_base_url = self.config.db_base_url + '/db/data/transaction' - data = dbu.statement_set_to_REST_form(op.statement_set) + statement_set = dbu.statement_set_to_REST_form(op.statement_set) try: op.begin(tx_base_url) tx_url = "{0}/{1}".format(tx_base_url, op.tx_id) - ret_tx = dbu.post_neo4j(tx_url, data) + ret_tx = dbu.post_neo4j(tx_url, statement_set) op.commit() + self.log_committed_queries(statement_set) return op.on_success(ret_tx) except Exception as e: log.error(e.message) -- cgit v1.3.1 From 11d9e3696691d096ed9cd5b2c41810414d960f74 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 24 Sep 2014 02:01:37 +0300 Subject: add DBO_load_node_set_by_id DB op --- src-py/db_controller.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index d7492e95..de3133a4 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -141,6 +141,20 @@ class DBO_load_node_id_set(DB_op): log.debug('loaded node id set: ' + str(id_set)) return id_set +class DBO_load_node_set_by_id(DB_op): + """ + load a set of nodes by ids + """ + + def __init__(self, id_set): + super(DBO_load_node_set_by_id, self).__init__() + q = "match (n) where n.id in {id_list} return n" + self.add_statement(q, { 'id_list': id_set}) + + def on_success(self, data): + log.debug('loaded node set: ' + str(data)) + return data + class DB_Controller: """ neo4j DB controller -- cgit v1.3.1 From c061b63a9bcc347854e1615b95eff85616b136b1 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 28 Sep 2014 13:21:26 +0300 Subject: distinguish between loading by DB id and by id attribute: - DBO_load_node_set_by_attribute - DBO_load_node_set_by_id_attribute (convenience op) --- src-py/db_controller.py | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index de3133a4..3ccc8bfd 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -76,14 +76,14 @@ class DB_op(object): pass class DBO_add_node_set(DB_op): - """ - DB op: add node set - - @param node_map: node-type to node list map - @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas - - """ def __init__(self, node_map, input_to_DB_property_map=lambda _: _): + """ + DB op: add node set + + @param node_map: node-type to node list map + @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas + + """ super(DBO_add_node_set, self).__init__() self.node_map = node_map @@ -107,10 +107,10 @@ class DBO_add_node_set(DB_op): return id_set class DBO_load_node_id_set(DB_op): - """ - load node id set, filter by type / properties - """ def __init__(self, filter_type, filter_prop=None): + """ + load node DB id set, filter by type / properties + """ super(DBO_load_node_id_set, self).__init__() # build where clause if necessary @@ -141,20 +141,29 @@ class DBO_load_node_id_set(DB_op): log.debug('loaded node id set: ' + str(id_set)) return id_set -class DBO_load_node_set_by_id(DB_op): - """ - load a set of nodes by ids - """ +class DBO_load_node_set_by_attribute(DB_op): - def __init__(self, id_set): - super(DBO_load_node_set_by_id, self).__init__() - q = "match (n) where n.id in {id_list} return n" - self.add_statement(q, { 'id_list': id_set}) + def __init__(self, attr_name, attr_set): + """ + load a set of nodes whose attr_name is in attr_set + + @return: loaded node set or an empty set if no match was found + """ + super(DBO_load_node_set_by_attribute, self).__init__() + q = "match (n) where n.{0} in {{attr_list}} return n".format(attr_name) + self.add_statement(q, { 'attr_list': attr_set}) def on_success(self, data): log.debug('loaded node set: ' + str(data)) return data +class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): + def __init__(self, id_set): + """ + convenience op for load a set of nodes by their 'id' attribute != DB node id + """ + super(DBO_load_node_set_by_id_attribute, self).__init__('id', id_set) + class DB_Controller: """ neo4j DB controller -- cgit v1.3.1 From a170bda6cea7810ee4fea0738b2144ac00bba1f1 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 28 Sep 2014 13:36:38 +0300 Subject: parse_single_query_response_data: assist in single query result parsing --- src-py/db_controller.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 3ccc8bfd..c9305fb9 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -69,6 +69,17 @@ class DB_op(object): def statement_set(self): return self.id_to_statement_map.values() + def parse_single_query_response_data(self, q, data): + """ + DB op can issue complex sets of quries all at once - this helper method + assists in parsing response data from a single query. + """ + ret = [] + r_0 = data['results'][0] + for row in r_0['data']: + ret.append(row['row'][0]) + return ret + def on_success(self, data): pass @@ -155,7 +166,7 @@ class DBO_load_node_set_by_attribute(DB_op): def on_success(self, data): log.debug('loaded node set: ' + str(data)) - return data + return self.parse_single_query_response_data(self.statement_set[0], data) class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): def __init__(self, id_set): -- cgit v1.3.1 From f93e7846097e73d5f021594af16264fe7ebdf9d0 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 28 Sep 2014 14:31:25 +0300 Subject: DBO_load_node_set_by_DB_id & test --- src-py/db_controller.py | 24 ++++++++++++++++++++---- src-py/test_db_controller.py | 8 ++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index c9305fb9..a6f738bd 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -93,7 +93,6 @@ class DBO_add_node_set(DB_op): @param node_map: node-type to node list map @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas - """ super(DBO_add_node_set, self).__init__() self.node_map = node_map @@ -152,6 +151,21 @@ class DBO_load_node_id_set(DB_op): log.debug('loaded node id set: ' + str(id_set)) return id_set +class DBO_load_node_set_by_DB_id(DB_op): + def __init__(self, id_set): + """ + load a set of nodes whose DB id is in id_set + + @return: loaded node set or an empty set if no match was found + """ + super(DBO_load_node_set_by_DB_id, self).__init__() + q = "match (n) where id(n) in {id_set} return n" + self.add_statement(q, { 'id_set': id_set}) + + def on_success(self, data): + log.debug('loaded node set: ' + str(data)) + return self.parse_single_query_response_data(self.statement_set[0], data) + class DBO_load_node_set_by_attribute(DB_op): def __init__(self, attr_name, attr_set): @@ -161,8 +175,8 @@ class DBO_load_node_set_by_attribute(DB_op): @return: loaded node set or an empty set if no match was found """ super(DBO_load_node_set_by_attribute, self).__init__() - q = "match (n) where n.{0} in {{attr_list}} return n".format(attr_name) - self.add_statement(q, { 'attr_list': attr_set}) + q = "match (n) where n.{0} in {{attr_set}} return n".format(attr_name) + self.add_statement(q, { 'attr_set': attr_set}) def on_success(self, data): log.debug('loaded node set: ' + str(data)) @@ -213,4 +227,6 @@ class DB_Controller: """ @deprecated: use transaction based api """ - self.post_neo4j('/db/data/cypher', {"query" : q}) + + # call post and not dbu.post_neo4j to avoid response key errors + dbu.post(self.config.db_base_url + '/db/data/cypher', {"query" : q}) diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 6e3a42ff..3f4db98f 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -33,6 +33,14 @@ class TestDBController(unittest.TestCase): self.db_ctl.exec_cypher_query('match ()-[r]-() delete r') self.db_ctl.exec_cypher_query('match (n) delete n') + def test_node_DB_id_lifecycle(self): + """ + test node DB id life cycle + """ + id_set = self.db_ctl.exec_op(dbc.DBO_add_node_set(self.n_map)) + n_set = self.db_ctl.exec_op(dbc.DBO_load_node_set_by_DB_id(id_set)) + self.assertEqual(len(n_set), len(id_set), 'incorrect result size') + def test_node_lifecycle(self): """ test node commit & load -- cgit v1.3.1 From 25e6bbb5e0796cf9a382e252cc47834fa8415014 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 28 Sep 2014 23:34:33 +0300 Subject: DBO_add_node_set arg type sanity checking --- src-py/db_controller.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index a6f738bd..8adcc9eb 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -69,7 +69,7 @@ class DB_op(object): def statement_set(self): return self.id_to_statement_map.values() - def parse_single_query_response_data(self, q, data): + def extract_single_query_response_data(self, q, data): """ DB op can issue complex sets of quries all at once - this helper method assists in parsing response data from a single query. @@ -95,6 +95,11 @@ class DBO_add_node_set(DB_op): @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas """ super(DBO_add_node_set, self).__init__() + + for k, v in node_map.iteritems(): # do some type sanity checking + assert isinstance(k, str) + assert isinstance(v, list) + self.node_map = node_map for type, n_set in self.node_map.items(): @@ -164,7 +169,7 @@ class DBO_load_node_set_by_DB_id(DB_op): def on_success(self, data): log.debug('loaded node set: ' + str(data)) - return self.parse_single_query_response_data(self.statement_set[0], data) + return self.extract_single_query_response_data(self.statement_set[0], data) class DBO_load_node_set_by_attribute(DB_op): @@ -174,19 +179,23 @@ class DBO_load_node_set_by_attribute(DB_op): @return: loaded node set or an empty set if no match was found """ + assert isinstance(attr_set, list) + super(DBO_load_node_set_by_attribute, self).__init__() q = "match (n) where n.{0} in {{attr_set}} return n".format(attr_name) self.add_statement(q, { 'attr_set': attr_set}) def on_success(self, data): log.debug('loaded node set: ' + str(data)) - return self.parse_single_query_response_data(self.statement_set[0], data) + return self.extract_single_query_response_data(self.statement_set[0], data) class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): def __init__(self, id_set): """ convenience op for load a set of nodes by their 'id' attribute != DB node id """ + assert isinstance(id_set, list) + super(DBO_load_node_set_by_id_attribute, self).__init__('id', id_set) class DB_Controller: -- cgit v1.3.1 From 064131c6254178446dd780f2dbc26af8836d65fc Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 29 Sep 2014 17:07:45 +0300 Subject: switch to app/json post argument content type --- src-py/db_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 8adcc9eb..67bb64d1 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -97,7 +97,7 @@ class DBO_add_node_set(DB_op): super(DBO_add_node_set, self).__init__() for k, v in node_map.iteritems(): # do some type sanity checking - assert isinstance(k, str) + assert isinstance(k, basestring) assert isinstance(v, list) self.node_map = node_map @@ -134,7 +134,7 @@ class DBO_load_node_id_set(DB_op): filter_prop_arr = [] for k, v in filter_prop: v_str = str(v) - if isinstance(v, str): + if isinstance(v, basestring): # quote string values v_str = "'{0}'".format(v_str) filter_prop_arr.append("n.{0} = {1} and ".format(k, v_str)) -- cgit v1.3.1 From 78bf830f0a48fa3dd9d2e956b94eebf1d0eac7b2 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 5 Oct 2014 20:50:58 +0200 Subject: DB_op refactor: mv begin(), commit() functions to db_controller bringing us closer towards op reusability with both REST/Embedded DB instances --- src-py/db_controller.py | 82 ++++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 35 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 67bb64d1..309ba823 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -23,38 +23,11 @@ class DB_op(object): self.tx_id = None self.tx_commit_url = None # cached from response to tx begin - def begin(self, tx_open_url): - try: - # - # [!] neo4j seems picky about receiving an additional empty statement list - # - data = data = dbu.statement_set_to_REST_form([]) - ret = dbu.post_neo4j(tx_open_url, data) - tx_commit_url = ret['commit'] - self.parse_tx_id(tx_commit_url) - self.tx_commit_url = tx_commit_url - - log.debug('tx-open: id: {0}, commit-url: {1}'.format(self.tx_id, tx_commit_url)) - except Exception as e: - raise Exception('failed to open transaction:' + e.message) - def parse_tx_id(self, tx_commit_url): m = re.search('/(?P\d+)/commit$', tx_commit_url) id_str = m.group('id') self.tx_id = int(id_str) - def commit(self): - try: - # - # [!] neo4j seems picky about receiving an additional empty statement list - # - data = dbu.statement_set_to_REST_form([]) - ret = dbu.post(self.tx_commit_url, data) - except Exception as e: - raise Exception('failed to commit transaction:' + e.message) - - log.debug('tx-commit: id: {0}, commit-url: {1}'.format(self.tx_id, self.tx_commit_url)) - def add_statement(self, cypher_query, params={}): """ add a DB query language statement @@ -204,24 +177,63 @@ class DB_Controller: """ def __init__(self, config): self.config = config + self.tx_base_url = self.config.db_base_url + '/db/data/transaction' def log_committed_queries(self, statement_set): for sp_dict in statement_set['statements']: log.debug('\tq: {0}'.format(sp_dict['statement'])) + def __begin_tx(self, op): + tx_open_url = self.tx_base_url + + try: + # + # [!] neo4j seems picky about receiving an additional empty statement list + # + data = data = dbu.statement_set_to_REST_form([]) + ret = dbu.post_neo4j(tx_open_url, data) + tx_commit_url = ret['commit'] + op.parse_tx_id(tx_commit_url) + + log.debug('tx-open: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) + except Exception as e: + raise Exception('failed to open transaction:' + e.message) + + def __exex_op_statements(self, op): + tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) + statement_set = dbu.statement_set_to_REST_form(op.statement_set) + + try: + ret = dbu.post_neo4j(tx_url, statement_set) + self.log_committed_queries(statement_set) + return ret + except Exception as e: + raise Exception('failed exec op statements: err: {0}, url: {1}'.format(e.message, tx_url)) + + def __commit_tx(self, op): + tx_commit_url = "{0}/{1}/commit".format(self.tx_base_url, op.tx_id) + + try: + # + # [!] neo4j seems picky about receiving an additional empty statement list + # + data = dbu.statement_set_to_REST_form([]) + ret = dbu.post(tx_commit_url, data) + + log.debug('tx-commit: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) + + return ret + except Exception as e: + raise Exception('failed to commit transaction:' + e.message) + def exec_op(self, op): """ execute operation within a DB transaction """ - tx_base_url = self.config.db_base_url + '/db/data/transaction' - statement_set = dbu.statement_set_to_REST_form(op.statement_set) - try: - op.begin(tx_base_url) - tx_url = "{0}/{1}".format(tx_base_url, op.tx_id) - ret_tx = dbu.post_neo4j(tx_url, statement_set) - op.commit() - self.log_committed_queries(statement_set) + self.__begin_tx(op) + ret_tx = self.__exex_op_statements(op) + ret_commit = self.__commit_tx(op) return op.on_success(ret_tx) except Exception as e: log.error(e.message) -- cgit v1.3.1 From 82f927d4ca2ca671afccc3a5efd1e73e2e89e9ca Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 5 Oct 2014 20:51:57 +0200 Subject: fix DBO_load_node_set_by_attribute filter query part construction --- src-py/db_controller.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 309ba823..04312eaa 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -146,17 +146,32 @@ class DBO_load_node_set_by_DB_id(DB_op): class DBO_load_node_set_by_attribute(DB_op): - def __init__(self, attr_name, attr_set): + def __init__(self, filter_attr_map): """ - load a set of nodes whose attr_name is in attr_set + load a set of nodes according to filter_attr_map + @param filter_attr_map: is a filter_key to filter_value_set map of + attributes to match against, eg.: + { 'id':[0,1], 'color: ['red','blue'] } @return: loaded node set or an empty set if no match was found """ - assert isinstance(attr_set, list) - + + # type sanity checks + assert isinstance(filter_attr_map, dict) + assert len(filter_attr_map) > 0 + for k, v in filter_attr_map.items(): + assert isinstance(k, basestring) + assert isinstance(v, list) + + filter_arr = [] + for k, v in filter_attr_map.items(): + f_attr = "n.{0} in {1}".format(k, v) + filter_arr.append(f_attr) + filter_str = "where {0}".format(' and '.join(filter_arr)) + super(DBO_load_node_set_by_attribute, self).__init__() - q = "match (n) where n.{0} in {{attr_set}} return n".format(attr_name) - self.add_statement(q, { 'attr_set': attr_set}) + q = "match (n) {0} return n".format(filter_str) + self.add_statement(q, { 'attr_set': filter_str}) def on_success(self, data): log.debug('loaded node set: ' + str(data)) -- cgit v1.3.1 From c1ff46d12a3a9d43d84bb337fa1795d0795a7a15 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 5 Oct 2014 20:53:23 +0200 Subject: misc - switch to iteritems() --- src-py/db_controller.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 04312eaa..d72c7bc4 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -68,11 +68,11 @@ class DBO_add_node_set(DB_op): @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas """ super(DBO_add_node_set, self).__init__() - - for k, v in node_map.iteritems(): # do some type sanity checking + + for k, v in node_map.iteritems(): # do some type sanity checking assert isinstance(k, basestring) assert isinstance(v, list) - + self.node_map = node_map for type, n_set in self.node_map.items(): -- cgit v1.3.1 From 4cce2e3737a78ffca41ada15b3f0d38968b60900 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 5 Oct 2014 20:54:00 +0200 Subject: adapt DBO_load_node_set_by_id_attribute to DBO_load_node_set_by_attribute --- src-py/db_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index d72c7bc4..2c3eb873 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -180,11 +180,11 @@ class DBO_load_node_set_by_attribute(DB_op): class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): def __init__(self, id_set): """ - convenience op for load a set of nodes by their 'id' attribute != DB node id + convenience op: load a set of nodes by their 'id' attribute != DB node id """ assert isinstance(id_set, list) - super(DBO_load_node_set_by_id_attribute, self).__init__('id', id_set) + super(DBO_load_node_set_by_id_attribute, self).__init__({'id': id_set}) class DB_Controller: """ -- cgit v1.3.1 From c52c99169826846bc73d53b189c753507e91a4c9 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 5 Oct 2014 21:13:52 +0200 Subject: refactor DB_Driver_REST, add DB_Driver_Embedded stub --- src-py/db_controller.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 2c3eb873..aaed2977 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -186,19 +186,11 @@ class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): super(DBO_load_node_set_by_id_attribute, self).__init__({'id': id_set}) -class DB_Controller: - """ - neo4j DB controller - """ - def __init__(self, config): - self.config = config - self.tx_base_url = self.config.db_base_url + '/db/data/transaction' +class DB_Driver_REST: + def __init__(self, db_base_url): + self.tx_base_url = db_base_url + '/db/data/transaction' - def log_committed_queries(self, statement_set): - for sp_dict in statement_set['statements']: - log.debug('\tq: {0}'.format(sp_dict['statement'])) - - def __begin_tx(self, op): + def begin_tx(self, op): tx_open_url = self.tx_base_url try: @@ -214,7 +206,7 @@ class DB_Controller: except Exception as e: raise Exception('failed to open transaction:' + e.message) - def __exex_op_statements(self, op): + def exex_op_statements(self, op): tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) statement_set = dbu.statement_set_to_REST_form(op.statement_set) @@ -225,7 +217,7 @@ class DB_Controller: except Exception as e: raise Exception('failed exec op statements: err: {0}, url: {1}'.format(e.message, tx_url)) - def __commit_tx(self, op): + def commit_tx(self, op): tx_commit_url = "{0}/{1}/commit".format(self.tx_base_url, op.tx_id) try: @@ -241,14 +233,31 @@ class DB_Controller: except Exception as e: raise Exception('failed to commit transaction:' + e.message) + def log_committed_queries(self, statement_set): + for sp_dict in statement_set['statements']: + log.debug('\tq: {0}'.format(sp_dict['statement'])) + +class DB_Driver_Embedded: + pass + +class DB_Controller: + """ + neo4j DB controller + """ + def __init__(self, config, db_driver=None): + self.config = config + if not db_driver: + db_driver = DB_Driver_REST(self.config.db_base_url) + self.db_driver = db_driver + def exec_op(self, op): """ execute operation within a DB transaction """ try: - self.__begin_tx(op) - ret_tx = self.__exex_op_statements(op) - ret_commit = self.__commit_tx(op) + self.db_driver.begin_tx(op) + ret_tx = self.db_driver.exex_op_statements(op) + ret_commit = self.db_driver.commit_tx(op) return op.on_success(ret_tx) except Exception as e: log.error(e.message) -- cgit v1.3.1 From c176cbea614d7e4d93f16c1479b97fd2641db12b Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 6 Oct 2014 17:29:59 +0300 Subject: use cypher query parameter place-holders: - each attribute filter set in passed as a query parameter - avoid string encoding issues when attempting non-parameterized queries --- src-py/db_controller.py | 52 ++++++++++++++++++++++++++++++++++++-------- src-py/test_db_controller.py | 12 +++++----- 2 files changed, 50 insertions(+), 14 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index aaed2977..93e1c553 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -38,6 +38,12 @@ class DB_op(object): self.s_id = self.s_id + 1 return ret + def __iter__(self): + # TODO impl + pass + # for k, v in self.id_to_statement_map: + # yield {k, v, None} + @property def statement_set(self): return self.id_to_statement_map.values() @@ -165,13 +171,16 @@ class DBO_load_node_set_by_attribute(DB_op): filter_arr = [] for k, v in filter_attr_map.items(): - f_attr = "n.{0} in {1}".format(k, v) + # create a cypher query parameter place holder for each attr set + # eg. n.foo in {foo}, where foo is passed as a query parameter + f_attr = "n.{0} in {{{0}}}".format(k, v) filter_arr.append(f_attr) + filter_str = "where {0}".format(' and '.join(filter_arr)) super(DBO_load_node_set_by_attribute, self).__init__() q = "match (n) {0} return n".format(filter_str) - self.add_statement(q, { 'attr_set': filter_str}) + self.add_statement(q, params=filter_attr_map) def on_success(self, data): log.debug('loaded node set: ' + str(data)) @@ -186,7 +195,10 @@ class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): super(DBO_load_node_set_by_id_attribute, self).__init__({'id': id_set}) -class DB_Driver_REST: +class DB_Driver_Base(): + pass + +class DB_Driver_REST(DB_Driver_Base): def __init__(self, db_base_url): self.tx_base_url = db_base_url + '/db/data/transaction' @@ -237,18 +249,40 @@ class DB_Driver_REST: for sp_dict in statement_set['statements']: log.debug('\tq: {0}'.format(sp_dict['statement'])) -class DB_Driver_Embedded: - pass +class DB_Driver_Embedded(DB_Driver_Base): + def __init__(self, db_base_url): + self.tx_base_url = db_base_url + '/db/data/transaction' + + from org.rhizi.db.neo4j.util import EmbeddedNeo4j + self.edb = EmbeddedNeo4j.createDb() + self.edb.createDb() + + def begin_tx(self, op): + pass + + def exex_op_statements(self, op): + s_set = op.statement_set + self.edb.executeCypherQury() + + def commit_tx(self, op): + pass + + def log_committed_queries(self, statement_set): + for sp_dict in statement_set['statements']: + log.debug('\tq: {0}'.format(sp_dict['statement'])) + class DB_Controller: """ neo4j DB controller """ - def __init__(self, config, db_driver=None): + def __init__(self, config, db_driver_class=None): self.config = config - if not db_driver: - db_driver = DB_Driver_REST(self.config.db_base_url) - self.db_driver = db_driver + if not db_driver_class: + self.db_driver = DB_Driver_REST(self.config.db_base_url) + else: + self.db_driver = db_driver_class() + assert isinstance(self.db_driver, DB_Driver_Base) def exec_op(self, op): """ diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 65f0231f..f64c47ce 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -22,18 +22,19 @@ class TestDBController(unittest.TestCase): @classmethod def setUpClass(self): cfg = Config.init_from_file('res/etc/rhizi-server.conf') - self.log = logging.getLogger('rhizi') self.db_ctl = dbc.DB_Controller(cfg) + self.db_ctl.exec_op(dbc.DBO_add_node_set(self.n_map)) + self.log = logging.getLogger('rhizi') def setUp(self): - self.db_ctl.exec_op(dbc.DBO_add_node_set(self.n_map)) + pass def test_load_node_set_by_attribute(self): - filter_map = { 'name': ['Bob', 'Judo'], + filter_map = { 'name': ['Bob', u'Judo'], 'age': [128] } n_set = self.db_ctl.exec_op(dbc.DBO_load_node_set_by_attribute(filter_map)) self.assertEqual(len(n_set), 1) - + filter_map = { 'age': [128, 256, 404] } n_set = self.db_ctl.exec_op(dbc.DBO_load_node_set_by_attribute(filter_map)) self.assertEqual(len(n_set), 2) @@ -46,7 +47,8 @@ class TestDBController(unittest.TestCase): """ test node DB id life cycle """ - id_set = self.db_ctl.exec_op(dbc.DBO_add_node_set(self.n_map)) + id_set = self.db_ctl.exec_op(dbc.DBO_add_node_set({'Person': [{'name': 'John Doe', 'id': 'jdoe_00'}, + {'name': 'John Doe', 'id': 'jdoe_01'}]})) n_set = self.db_ctl.exec_op(dbc.DBO_load_node_set_by_DB_id(id_set)) self.assertEqual(len(n_set), len(id_set), 'incorrect result size') -- cgit v1.3.1 From 2e080bafa5b4d30361e73652c2b12f6d4cfb8705 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 6 Oct 2014 17:55:11 +0300 Subject: DBO_load_node_id_set: fix cypher query --- src-py/db_controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 93e1c553..96b12079 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -102,6 +102,7 @@ class DBO_add_node_set(DB_op): class DBO_load_node_id_set(DB_op): def __init__(self, filter_type, filter_prop=None): + # TODO: mv type filter to DBO_load_node_set_by_attribute """ load node DB id set, filter by type / properties """ @@ -119,7 +120,7 @@ class DBO_load_node_id_set(DB_op): filter_prop_arr.append("n.{0} = {1} and ".format(k, v_str)) filter_prop_str = " where " + " and ".join(filter_prop_arr) - q = "match (n:{0}){1} return id(n)".format(filter_type, filter_prop_str) + q = "match (n:{0} {1}) return id(n)".format(filter_type, filter_prop_str) self.add_statement(q) def on_success(self, data): -- cgit v1.3.1 From b7ac56cbe38fbccbb782b42c07bb5b389e65a433 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 6 Oct 2014 18:15:10 +0300 Subject: common DB_op __type_check_filter_attr_map() --- src-py/db_controller.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 96b12079..b4ce0fc5 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -44,6 +44,16 @@ class DB_op(object): # for k, v in self.id_to_statement_map: # yield {k, v, None} + def __type_check_filter_attr_map(self, filter_attr_map): + """ + # type sanity check an attribute filter map + """ + assert isinstance(filter_attr_map, dict) + assert len(filter_attr_map) > 0 + for k, v in filter_attr_map.items(): + assert isinstance(k, basestring) + assert isinstance(v, list) + @property def statement_set(self): return self.id_to_statement_map.values() -- cgit v1.3.1 From e4fbde3cea8a5febda828b12927a3759d430606d Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 6 Oct 2014 18:49:14 +0300 Subject: handle empty filter maps --- src-py/db_controller.py | 42 ++++++++++++++++++++++-------------------- src-py/neo4j_util.py | 5 +++++ 2 files changed, 27 insertions(+), 20 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index b4ce0fc5..dcc22f7c 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -112,7 +112,7 @@ class DBO_add_node_set(DB_op): class DBO_load_node_id_set(DB_op): def __init__(self, filter_type, filter_prop=None): - # TODO: mv type filter to DBO_load_node_set_by_attribute + # TODO: mv type filter to DBO_load_node_set """ load node DB id set, filter by type / properties """ @@ -161,35 +161,23 @@ class DBO_load_node_set_by_DB_id(DB_op): log.debug('loaded node set: ' + str(data)) return self.extract_single_query_response_data(self.statement_set[0], data) -class DBO_load_node_set_by_attribute(DB_op): +class DBO_load_node_set(DB_op): - def __init__(self, filter_attr_map): + def __init__(self, filter_type=None, filter_attr_map=None): """ load a set of nodes according to filter_attr_map @param filter_attr_map: is a filter_key to filter_value_set map of attributes to match against, eg.: { 'id':[0,1], 'color: ['red','blue'] } + @param filter_type: node type filter @return: loaded node set or an empty set if no match was found """ - # type sanity checks - assert isinstance(filter_attr_map, dict) - assert len(filter_attr_map) > 0 - for k, v in filter_attr_map.items(): - assert isinstance(k, basestring) - assert isinstance(v, list) - - filter_arr = [] - for k, v in filter_attr_map.items(): - # create a cypher query parameter place holder for each attr set - # eg. n.foo in {foo}, where foo is passed as a query parameter - f_attr = "n.{0} in {{{0}}}".format(k, v) - filter_arr.append(f_attr) + self.__type_check_filter_attr_map(filter_attr_map) + filter_str = dbu.where_clause_from_filter_attr_map() - filter_str = "where {0}".format(' and '.join(filter_arr)) - - super(DBO_load_node_set_by_attribute, self).__init__() + super(DBO_load_node_set, self).__init__() q = "match (n) {0} return n".format(filter_str) self.add_statement(q, params=filter_attr_map) @@ -197,7 +185,7 @@ class DBO_load_node_set_by_attribute(DB_op): log.debug('loaded node set: ' + str(data)) return self.extract_single_query_response_data(self.statement_set[0], data) -class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): +class DBO_load_node_set_by_id_attribute(DBO_load_node_set): def __init__(self, id_set): """ convenience op: load a set of nodes by their 'id' attribute != DB node id @@ -206,6 +194,20 @@ class DBO_load_node_set_by_id_attribute(DBO_load_node_set_by_attribute): super(DBO_load_node_set_by_id_attribute, self).__init__({'id': id_set}) +class DBO_load_link_id_set(DB_op): + def __init__(self, filter_type=None, filter_attr_map=None): + """ + load a set of link ids + + @param filter_type: link type filter + @param filter_attr_map: is a filter_key to filter_value_set map of + attributes to match link properties against + @return: a set of loaded link ids + """ + self.__type_check_filter_attr_map(filter_attr_map) + filter_str = dbu.where_clause_from_filter_attr_map() + + class DB_Driver_Base(): pass diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 929324fb..c340e15a 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -54,7 +54,12 @@ def where_clause_from_filter_attr_map(filter_attr_map, node_param_name="n"): convert a filter attribute map to a parameterized Cypher where clause, eg. in: { 'att_foo': [ 'a', 'b' ], 'att_goo': [1,2] } out: where n.att_foo in {att_foo} and n.att_goo in {att_goo} ... + + @param filter_attr_map: may be None or empty """ + if not filter_attr_map: + return "" + filter_arr = [] for k in filter_attr_map.keys(): # create a cypher query parameter place holder for each attr set -- cgit v1.3.1 From 4717f01ed33b26f6c0fbbf2cb83fbf681d14da41 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 7 Oct 2014 18:08:20 +0200 Subject: mv type sanity checks to neo4j_util --- src-py/db_controller.py | 12 ------------ src-py/neo4j_util.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index dcc22f7c..b76de5fe 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -44,16 +44,6 @@ class DB_op(object): # for k, v in self.id_to_statement_map: # yield {k, v, None} - def __type_check_filter_attr_map(self, filter_attr_map): - """ - # type sanity check an attribute filter map - """ - assert isinstance(filter_attr_map, dict) - assert len(filter_attr_map) > 0 - for k, v in filter_attr_map.items(): - assert isinstance(k, basestring) - assert isinstance(v, list) - @property def statement_set(self): return self.id_to_statement_map.values() @@ -174,7 +164,6 @@ class DBO_load_node_set(DB_op): @return: loaded node set or an empty set if no match was found """ - self.__type_check_filter_attr_map(filter_attr_map) filter_str = dbu.where_clause_from_filter_attr_map() super(DBO_load_node_set, self).__init__() @@ -204,7 +193,6 @@ class DBO_load_link_id_set(DB_op): attributes to match link properties against @return: a set of loaded link ids """ - self.__type_check_filter_attr_map(filter_attr_map) filter_str = dbu.where_clause_from_filter_attr_map() diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index c340e15a..2d870fcc 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -69,3 +69,15 @@ def where_clause_from_filter_attr_map(filter_attr_map, node_param_name="n"): filter_str = "where {0}".format(' and '.join(filter_arr)) return filter_str +def __type_check_link(link): + assert link.has_key('__src') + assert link.has_key('__dst') + +def __type_check_filter_attr_map(filter_attr_map): + """ + # type sanity check an attribute filter map + """ + assert isinstance(filter_attr_map, dict) + for k, v in filter_attr_map.items(): + assert isinstance(k, basestring) + assert isinstance(v, list) -- cgit v1.3.1 From 7ca40693c02664c11f3304f5da4d18e137e7b423 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 7 Oct 2014 18:08:35 +0200 Subject: rm old DBO_load_node_id_set --- src-py/db_controller.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index b76de5fe..8ca3a642 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -100,41 +100,6 @@ class DBO_add_node_set(DB_op): log.debug('node-set added: ids: ' + str(id_set)) return id_set -class DBO_load_node_id_set(DB_op): - def __init__(self, filter_type, filter_prop=None): - # TODO: mv type filter to DBO_load_node_set - """ - load node DB id set, filter by type / properties - """ - super(DBO_load_node_id_set, self).__init__() - - # build where clause if necessary - filter_prop_str = "" - if filter_prop: - filter_prop_arr = [] - for k, v in filter_prop: - v_str = str(v) - if isinstance(v, basestring): - # quote string values - v_str = "'{0}'".format(v_str) - filter_prop_arr.append("n.{0} = {1} and ".format(k, v_str)) - filter_prop_str = " where " + " and ".join(filter_prop_arr) - - q = "match (n:{0} {1}) return id(n)".format(filter_type, filter_prop_str) - self.add_statement(q) - - def on_success(self, data): - # [!] fragile - parse results - # sample input: dict: {u'errors': [], u'results': [{u'data': [{u'row': [20]}], u'columns': [u'id(n)']}]} - id_set = [] - for r in data['results']: - columns = r['columns'] - for k in r['data']: - nid = k['row'][0] - id_set.append(nid) - - log.debug('loaded node id set: ' + str(id_set)) - return id_set class DBO_load_node_set_by_DB_id(DB_op): def __init__(self, id_set): -- cgit v1.3.1 From 337e0d2ce4706e8ebd176b7a7e6e5f82d461e007 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 7 Oct 2014 21:19:36 +0200 Subject: DB_op statement __iter__ --- src-py/db_controller.py | 18 ++++++++++-------- src-py/test_db_controller.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 8ca3a642..52913ac5 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -39,14 +39,16 @@ class DB_op(object): return ret def __iter__(self): - # TODO impl - pass - # for k, v in self.id_to_statement_map: - # yield {k, v, None} - - @property - def statement_set(self): - return self.id_to_statement_map.values() + """ + iterate over (statement_index, statement, statement_result) + note: statement_index is zero based + + TODO: support statement_result + """ + i = 0 + for s in self.statement_set: + yield (i, s, None) + i = i + 1 def extract_single_query_response_data(self, q, data): """ diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index f64c47ce..f7092e6d 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -29,6 +29,20 @@ class TestDBController(unittest.TestCase): def setUp(self): pass + def test_db_op_statement_iter(self): + s_arr = ['match (n) return n', + 'create (b:Book {\'title\': \'foo\'}) return b'] + + db_op = dbc.DB_op() + db_op.add_statement(s_arr[0]) + db_op.add_statement(s_arr[1]) + + i = 0 + for s in db_op: + # access: second tuple item -> REST-form 'statement' key + self.assertEqual(s_arr[i], s[1]['statement']) + i = i + 1 + def test_load_node_set_by_attribute(self): filter_map = { 'name': ['Bob', u'Judo'], 'age': [128] } -- cgit v1.3.1 From abdd6499319a4857e2653697e8398fe13b141c39 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 13:09:54 +0200 Subject: change DB_Controller op execution logic - allow query-sets to partially succeed --- src-py/db_controller.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 52913ac5..180a6b37 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -61,10 +61,13 @@ class DB_op(object): ret.append(row['row'][0]) return ret - def on_success(self, data): + def parse_multi_statement_response_data(self, data): pass - def on_error(self): + def on_completion(self, data): + self.result_set = data['results'] + self.error_set = data['errors'] + pass class DBO_add_node_set(DB_op): @@ -89,7 +92,7 @@ class DBO_add_node_set(DB_op): p = {'prop_dict' : input_to_DB_property_map(n_prop_dict)} self.add_statement(q, p) - def on_success(self, data): + def on_completion(self, data): # [!] fragile - parse results # sample input: dict: {u'errors': [], u'results': [{u'data': [{u'row': [20]}], u'columns': [u'id(n)']}]} id_set = [] @@ -114,9 +117,9 @@ class DBO_load_node_set_by_DB_id(DB_op): q = "match (n) where id(n) in {id_set} return n" self.add_statement(q, { 'id_set': id_set}) - def on_success(self, data): + def on_completion(self, data): log.debug('loaded node set: ' + str(data)) - return self.extract_single_query_response_data(self.statement_set[0], data) + return self.parse_single_query_response_data(data) class DBO_load_node_set(DB_op): @@ -137,9 +140,11 @@ class DBO_load_node_set(DB_op): q = "match (n) {0} return n".format(filter_str) self.add_statement(q, params=filter_attr_map) - def on_success(self, data): - log.debug('loaded node set: ' + str(data)) - return self.extract_single_query_response_data(self.statement_set[0], data) + self.add_statement(q, q_params) + + def on_completion(self, data): + log.debug('loaded id-set: ' + str(data)) + return self.parse_single_query_response_data(data) class DBO_load_node_set_by_id_attribute(DBO_load_node_set): def __init__(self, id_set): @@ -260,11 +265,12 @@ class DB_Controller: self.db_driver.begin_tx(op) ret_tx = self.db_driver.exex_op_statements(op) ret_commit = self.db_driver.commit_tx(op) - return op.on_success(ret_tx) + return op.on_completion(ret_tx) except Exception as e: + # here we watch for IOExecptions, etc - not db errors + # these are returned in the db response itself log.error(e.message) log.error(traceback.print_exc()) - op.on_error() def create_db_op(self, f_work, f_cont): ret = DB_op(f_work, f_cont) -- cgit v1.3.1 From 6a5c1acbeca40a374736d657d48beb33f0d32b27 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 14:40:12 +0200 Subject: support result_set iteration post DB_op execution --- src-py/db_controller.py | 17 ++++++++++++----- src-py/test_db_controller.py | 25 +++++++++++++++++-------- 2 files changed, 29 insertions(+), 13 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 180a6b37..3969a2bb 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -40,15 +40,22 @@ class DB_op(object): def __iter__(self): """ - iterate over (statement_index, statement, statement_result) + iterate over (statement_index, statement, result, error) + where result & error are mutually exclusive + note: statement_index is zero based - TODO: support statement_result + TODO: handle partial iteration due to error_set being non-empty """ i = 0 - for s in self.statement_set: - yield (i, s, None) - i = i + 1 + if self.result_set: + for s in self.statement_set: + yield (i, s, self.result_set[i]) + i = i + 1 + else: + for s in self.statement_set: + yield (i, s, None) + i = i + 1 def extract_single_query_response_data(self, q, data): """ diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index e09548bb..7d96954e 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -29,18 +29,27 @@ class TestDBController(unittest.TestCase): def setUp(self): pass - def test_db_op_statement_iter(self): - s_arr = ['match (n) return n', - 'create (b:Book {\'title\': \'foo\'}) return b'] + def test_db_op_statement_iteration(self): + s_arr = ['create (b:Book {title: \'foo\'}) return b', + 'match (n) return n',] - db_op = dbc.DB_op() - db_op.add_statement(s_arr[0]) - db_op.add_statement(s_arr[1]) + op = dbc.DB_op() + op.add_statement(s_arr[0]) + op.add_statement(s_arr[1]) + + i = 0 + for s_id, s, r in op: + # access: second tuple item -> REST-form 'statement' key + self.assertEqual(s_arr[i], s['statement']) + self.assertEqual(None, r) + i = i + 1 + + self.db_ctl.exec_op(op) i = 0 - for s in db_op: + for s_id, s, r in op: # access: second tuple item -> REST-form 'statement' key - self.assertEqual(s_arr[i], s[1]['statement']) + self.assertNotEqual(None, r) i = i + 1 def test_load_node_set_by_attribute(self): -- cgit v1.3.1 From 8e20a72f5cd0464f673a374e5a3741a58932e770 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 15:17:17 +0200 Subject: return DB_result_set when iterating over DB_op results --- src-py/db_controller.py | 4 +++- src-py/neo4j_util.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 3969a2bb..32d5a436 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -9,6 +9,7 @@ import traceback import urllib2 import neo4j_util as dbu +from neo4j_util import DB_result_set log = logging.getLogger('rhizi') @@ -50,7 +51,8 @@ class DB_op(object): i = 0 if self.result_set: for s in self.statement_set: - yield (i, s, self.result_set[i]) + rs = DB_result_set(self.result_set[i]) + yield (i, s, rs) i = i + 1 else: for s in self.statement_set: diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 6b78dc18..7c228dea 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -7,6 +7,17 @@ import urllib2 import model import string +class DB_result_set(object): + def __init__(self, data): + self.data = data + + def __iter__(self): + for db_row_dict in self.data['data']: + # example: dict: {u'row': [{u'title': u'foo'}]} + assert None != db_row_dict['row'][0] + + yield db_row_dict['row'][0] + class Cypher_String_Formatter(string.Formatter): """ Despite parameter support in Cypher, we sometimes do engage in query string building -- cgit v1.3.1 From de70bef3e3be880b86e08aba810940d9481ce64b Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 15:18:48 +0200 Subject: n=node:IndexName(Key={value}) DBO_add_link_set operation & test: - use DB_op.__iter__ to compile on_completion() return value - return id(link) values until we come up with a better link id scheme --- src-py/db_controller.py | 21 +++++++++++++++++++++ src-py/test_db_controller.py | 15 +++++++++++++++ 2 files changed, 36 insertions(+) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 32d5a436..bff3a45b 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -114,6 +114,27 @@ class DBO_add_node_set(DB_op): log.debug('node-set added: ids: ' + str(id_set)) return id_set +class DBO_add_link_set(DB_op): + def __init__(self, link_map): + """ + @param link_map: is a link-type to link-set map - see model.link + """ + super(DBO_add_link_set, self).__init__() + for q, q_params in db_util.gen_query_create_from_link_map(link_map): + self.add_statement(q, q_params) + + def on_completion(self, data): + super(DBO_add_link_set, self).on_completion(data) + + id_set = [] + for s_id, s, r_set in self: + for row in r_set: + # [!] fragile - parse results + lid = row + id_set.append(lid) + + log.debug('link-set added: ids: ' + str(id_set)) + return id_set class DBO_load_node_set_by_DB_id(DB_op): def __init__(self, id_set): diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 7d96954e..cfa8ee31 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -52,6 +52,21 @@ class TestDBController(unittest.TestCase): self.assertNotEqual(None, r) i = i + 1 + def test_add_link_set(self): + l_map = { 'Knows' : [{'__src': 'person_00', '__dst': 'skill_00'}, + {'__src': 'person_00', '__dst': 'skill_01'}] } + l_set = self.db_ctl.exec_op(dbc.DBO_add_link_set(l_map)) + self.assertEqual(len(l_set), 2) + + def test_load_node_set_by_type(self): + filter_type = 'Person' + n_set = self.db_ctl.exec_op(dbc.DBO_load_node_id_set(filter_type=filter_type)) + self.assertEqual(len(n_set), 2) + + filter_type = 'Nan_Type' + n_set = self.db_ctl.exec_op(dbc.DBO_load_node_id_set(filter_type=filter_type)) + self.assertEqual(len(n_set), 0) + def test_load_node_set_by_attribute(self): filter_map = { 'name': ['Bob', u'Judo'], 'age': [128] } -- cgit v1.3.1 From 749f4efb5cace86377606cf924378ed79d5071d2 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 15:36:10 +0200 Subject: parse_single_query_response_data - use __iter__ --- src-py/db_controller.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index bff3a45b..726445a6 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -59,15 +59,15 @@ class DB_op(object): yield (i, s, None) i = i + 1 - def extract_single_query_response_data(self, q, data): + def parse_single_query_response_data(self, data): """ DB op can issue complex sets of quries all at once - this helper method assists in parsing response data from a single query. """ ret = [] - r_0 = data['results'][0] - for row in r_0['data']: - ret.append(row['row'][0]) + for _, _, r_set in self: + for row in r_set: + ret.append(row) return ret def parse_multi_statement_response_data(self, data): -- cgit v1.3.1 From 0b06576fdbc76a5f208f0233dae47bb0707520d2 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 18:03:14 +0200 Subject: DBO_load_link_set_by_src_or_dst_id_attributes & test --- src-py/db_controller.py | 30 ++++++++++++++++++++++++++++-- src-py/test_db_controller.py | 13 +++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 726445a6..64d33e8a 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -176,14 +176,40 @@ class DBO_load_node_set(DB_op): log.debug('loaded id-set: ' + str(data)) return self.parse_single_query_response_data(data) -class DBO_load_node_set_by_id_attribute(DBO_load_node_set): +class DBO_load_node_set_by_id_attribute(DBO_load_node_id_set): def __init__(self, id_set): """ convenience op: load a set of nodes by their 'id' attribute != DB node id """ assert isinstance(id_set, list) - super(DBO_load_node_set_by_id_attribute, self).__init__({'id': id_set}) + super(DBO_load_node_set_by_id_attribute, self).__init__(filter_attr_map={'id': id_set}) + + +class DBO_load_link_set_by_src_or_dst_id_attributes(DB_op): + def __init__(self, src_id=None, dst_id=None): + """ + load an id-set of links by source/target id attributes != DB node id + """ + assert None != src_id or None != dst_id + + super(DBO_load_link_set_by_src_or_dst_id_attributes, self).__init__() + + if not src_id: + q = "match ()-[r]->({id: {dst_id}}) return r" + q_params = {'dst_id': dst_id} + elif not dst_id: + q = "match ({id: {src_id}})-[r]->() return r" + q_params = {'src_id': src_id} + else: + q = "match ({id: {src_id}})-[r]->({id: {dst_id}}) return r" + q_params = {'src_id': src_id, 'dst_id': dst_id} + + self.add_statement(q, q_params) + + def on_completion(self, data): + log.debug('loaded id-set: ' + str(data)) + return self.parse_single_query_response_data(data) class DBO_load_link_id_set(DB_op): def __init__(self, filter_type=None, filter_attr_map=None): diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index cfa8ee31..258b91c8 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -81,6 +81,19 @@ class TestDBController(unittest.TestCase): n_set = self.db_ctl.exec_op(dbc.DBO_load_node_set_by_id_attribute(['skill_00', 'person_01'])) self.assertEqual(len(n_set), 2) + def test_load_link_set_by_src_or_dst_id_attributes(self): + op = dbc.DBO_load_link_set_by_src_or_dst_id_attributes(src_id='person_00', dst_id='skill_00') + n_set = self.db_ctl.exec_op(op) + self.assertEqual(len(n_set), 1) + + op = dbc.DBO_load_link_set_by_src_or_dst_id_attributes(src_id='person_00') + n_set = self.db_ctl.exec_op(op) + self.assertEqual(len(n_set), 2) + + op = dbc.DBO_load_link_set_by_src_or_dst_id_attributes(dst_id='skill_00') + n_set = self.db_ctl.exec_op(op) + self.assertEqual(len(n_set), 1) + def test_node_DB_id_lifecycle(self): """ test node DB id life cycle -- cgit v1.3.1 From f5aaa150c41e07104913fc7e99015ada36e228ac Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 18:06:05 +0200 Subject: DBO_add_node_set - sync with code changes --- src-py/db_controller.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 64d33e8a..936009b5 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -80,35 +80,21 @@ class DB_op(object): pass class DBO_add_node_set(DB_op): - def __init__(self, node_map, input_to_DB_property_map=lambda _: _): + def __init__(self, node_map): """ DB op: add node set - @param node_map: node-type to node list map - @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas + @param node_map: node-type to node-set map """ super(DBO_add_node_set, self).__init__() - - for k, v in node_map.iteritems(): # do some type sanity checking - assert isinstance(k, basestring) - assert isinstance(v, list) - - self.node_map = node_map - - for type, n_set in self.node_map.items(): - q = "create (n:{0} {{prop_dict}}) return id(n)".format(type) - for n_prop_dict in n_set: - p = {'prop_dict' : input_to_DB_property_map(n_prop_dict)} - self.add_statement(q, p) + for q, q_param_set in db_util.gen_query_create_from_node_map(node_map): + self.add_statement(q, q_param_set) def on_completion(self, data): - # [!] fragile - parse results - # sample input: dict: {u'errors': [], u'results': [{u'data': [{u'row': [20]}], u'columns': [u'id(n)']}]} id_set = [] - for r in data['results']: - columns = r['columns'] - for k in r['data']: - nid = k['row'][0] + for _, _, r_set in self: + for row in r_set: + nid = row # [!] fragile id_set.append(nid) log.debug('node-set added: ids: ' + str(id_set)) -- cgit v1.3.1 From 1d8d5646897ca05f97bb33be5ced2b8643ae4a23 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 18:07:21 +0200 Subject: DB_op - result_set, error_set --- src-py/db_controller.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 936009b5..96de855d 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -19,8 +19,9 @@ class DB_op(object): """ def __init__(self): - self.s_id = 0 # statement id counter - self.id_to_statement_map = {} # zero based id to statement map + self.statement_set = [] + self.result_set = None + self.error_set = None self.tx_id = None self.tx_commit_url = None # cached from response to tx begin @@ -29,15 +30,14 @@ class DB_op(object): id_str = m.group('id') self.tx_id = int(id_str) - def add_statement(self, cypher_query, params={}): + def add_statement(self, query, query_params={}): """ add a DB query language statement - @return: statement id + @return: statement index (zero based) """ - ret = self.s_id - self.id_to_statement_map[self.s_id] = dbu.statement_to_REST_form(cypher_query, params) - self.s_id = self.s_id + 1 - return ret + s = db_util.statement_to_REST_form(query, query_params) + self.statement_set.append(s) + return len(self.statement_set) def __iter__(self): """ -- cgit v1.3.1 From a9abe5a017b01c40615e23a279d46bb33c240e3f Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 12 Oct 2014 19:10:38 +0200 Subject: DBO_load_link_id_set & test --- src-py/db_controller.py | 18 ++++++++++++++---- src-py/test_db_controller.py | 9 +++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 96de855d..1e80ff84 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -198,17 +198,27 @@ class DBO_load_link_set_by_src_or_dst_id_attributes(DB_op): return self.parse_single_query_response_data(data) class DBO_load_link_id_set(DB_op): - def __init__(self, filter_type=None, filter_attr_map=None): + def __init__(self, filter_type=None, filter_attr_map={}): """ - load a set of link ids + load an id-set of links @param filter_type: link type filter @param filter_attr_map: is a filter_key to filter_value_set map of attributes to match link properties against @return: a set of loaded link ids """ - filter_str = dbu.where_clause_from_filter_attr_map() - + super(DBO_load_link_id_set, self).__init__() + + q = "match ()-[r{filter_type} {filter_attr}]->() return id(r)" + q = cfmt(q, filter_type="" if not filter_type else ":" + filter_type) + q = cfmt(q, filter_attr=db_util.gen_clause_attr_filter_from_filter_attr_map(filter_attr_map)) + q_params = {k: v[0] for (k, v) in filter_attr_map.items()} # pass on only first value from each value set + + self.add_statement(q, q_params) + + def on_completion(self, data): + log.debug('loaded id-set: ' + str(data)) + return self.parse_single_query_response_data(data) class DB_Driver_Base(): pass diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 446d9288..176fce4b 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -94,6 +94,15 @@ class TestDBController(unittest.TestCase): n_set = self.db_ctl.exec_op(dbc.DBO_load_node_set_by_id_attribute(['skill_00', 'person_01'])) self.assertEqual(len(n_set), 2) + def test_load_link_set_by_type(self): + op = dbc.DBO_load_link_id_set(filter_type='Knows') + id_set = self.db_ctl.exec_op(op) + self.assertEqual(len(id_set), 2) + + op = dbc.DBO_load_link_id_set(filter_type='Nan_Type') + id_set = self.db_ctl.exec_op(op) + self.assertEqual(len(id_set), 0) + def test_load_link_set_by_src_or_dst_id_attributes(self): op = dbc.DBO_load_link_set_by_src_or_dst_id_attributes(src_id='person_00', dst_id='skill_00') n_set = self.db_ctl.exec_op(op) -- cgit v1.3.1 From 93873bd93ccc8463eaa3931e1efe7d30d1007e0b Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 15 Oct 2014 18:37:00 +0200 Subject: DB_composed_op --- src-py/db_controller.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 1e80ff84..9a6151ec 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -74,9 +74,33 @@ class DB_op(object): pass def on_completion(self, data): + pass + + def _assign_results_errors(self, data): self.result_set = data['results'] self.error_set = data['errors'] +class DB_composed_op(DB_op): + def __init__(self): + super(DB_composed_op, self).__init__() + self.sub_op_set = [] + + def add_statement(self, query, query_params={}): + assert False, "composed_op may not contain statements, only sub-ops" + + def add_sub_op(self, op): + self.sub_op_set.append(op) + + def __getattribute__(self, attr): + """ + intercept 'statement_set' attr get + """ + if attr == 'statement_set': + # construct a list comprehension composed of all sup_op statements + return [s for s_op in self.sub_op_set for s in s_op.statement_set] + + return object.__getattribute__(self, attr) + pass class DBO_add_node_set(DB_op): -- cgit v1.3.1 From cd4c8fc54cdd6d24ef6385a89afdec1c9d65e9cb Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 15 Oct 2014 18:38:53 +0200 Subject: DBO_topo_diff_commit & test --- src-py/db_controller.py | 22 ++++++++++++++++++++++ src-py/test_db_controller.py | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 9a6151ec..dcb70c10 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -101,6 +101,28 @@ class DB_composed_op(DB_op): return object.__getattribute__(self, attr) +class DBO_topo_diff_commit(DB_composed_op): + """ + commit a + """ + def __init__(self, topo_diff): + super(DBO_topo_diff_commit, self).__init__() + + # TODO rm link set + # TODO rm node set + assert not topo_diff.node_set_rm, 'unsupported' + assert not topo_diff.link_set_rm, 'unsupported' + + n_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.node_set_add) + l_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.link_set_add) + + op_n_add = DBO_add_node_set(n_add_map) + op_l_add = DBO_add_link_set(l_add_map) + + # [!] order critical + self.add_sub_op(op_n_add) + self.add_sub_op(op_l_add) + pass class DBO_add_node_set(DB_op): diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index b768898d..2820068e 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -160,6 +160,28 @@ class TestDBController(unittest.TestCase): n_set = self.db_ctl.exec_op(dbc.DBO_load_node_set_by_id_attribute([n_id])) self.assertEqual(len(n_set), 0) + def test_topo_diff_commit(self): + n_0_id = rand_id() + n_1_id = rand_id() + + n_set = [{'__type': 'T_test_topo_diff_commit', 'id': n_0_id }, + {'__type': 'T_test_topo_diff_commit', 'id': n_1_id }] + l_set = [{'__type': 'T_test_topo_diff_commit', '__src': n_0_id, '__dst': n_1_id}, + {'__type': 'T_test_topo_diff_commit', '__src': n_1_id, '__dst': n_0_id}] + + topo_diff = Topo_Diff(node_set_add=n_set, + link_set_add=l_set) + + op = dbc.DBO_topo_diff_commit(topo_diff) + self.assertEqual(len(op.statement_set), 3) # one parameterized node create. 2 link create + self.db_ctl.exec_op(op) + + id_set = self.db_ctl.exec_op(dbc.DBO_match_node_set_by_id_attribute([n_0_id, n_1_id])) + self.assertEqual(len(id_set), 2) + id_set = self.db_ctl.exec_op(dbc.DBO_match_link_set_by_src_or_dst_id_attributes(src_id=n_0_id, dst_id=n_1_id)) + self.assertEqual(len(id_set), 1) + id_set = self.db_ctl.exec_op(dbc.DBO_match_link_set_by_src_or_dst_id_attributes(src_id=n_1_id, dst_id=n_0_id)) + self.assertEqual(len(id_set), 1) def tearDown(self): pass if __name__ == "__main__": -- cgit v1.3.1 From 12b475ab0f131c6db8f5b28b7d5e5a89d11be9cf Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 15 Oct 2014 18:54:36 +0200 Subject: misc --- src-py/db_controller.py | 40 ++++++++++++++++++++-------------------- src-py/neo4j_util.py | 19 ++++++++++++------- 2 files changed, 32 insertions(+), 27 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index dcb70c10..cb52cc09 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -156,13 +156,10 @@ class DBO_add_link_set(DB_op): self.add_statement(q, q_params) def on_completion(self, data): - super(DBO_add_link_set, self).on_completion(data) - id_set = [] for s_id, s, r_set in self: for row in r_set: - # [!] fragile - parse results - lid = row + lid = row # [!] fragile id_set.append(lid) log.debug('link-set added: ids: ' + str(id_set)) @@ -183,7 +180,7 @@ class DBO_load_node_set_by_DB_id(DB_op): log.debug('loaded node set: ' + str(data)) return self.parse_single_query_response_data(data) -class DBO_load_node_set(DB_op): +class DBO_match_node_id_set(DB_op): def __init__(self, filter_type=None, filter_attr_map=None): """ @@ -208,24 +205,26 @@ class DBO_load_node_set(DB_op): log.debug('loaded id-set: ' + str(data)) return self.parse_single_query_response_data(data) -class DBO_load_node_set_by_id_attribute(DBO_load_node_id_set): +class DBO_match_node_set_by_id_attribute(DBO_match_node_id_set): def __init__(self, id_set): """ convenience op: load a set of nodes by their 'id' attribute != DB node id """ assert isinstance(id_set, list) - super(DBO_load_node_set_by_id_attribute, self).__init__(filter_attr_map={'id': id_set}) + super(DBO_match_node_set_by_id_attribute, self).__init__(filter_attr_map={'id': id_set}) -class DBO_load_link_set_by_src_or_dst_id_attributes(DB_op): +class DBO_match_link_set_by_src_or_dst_id_attributes(DB_op): def __init__(self, src_id=None, dst_id=None): """ - load an id-set of links by source/target id attributes != DB node id + match a set of links by source/target node id attributes + + @return: a set of loaded links """ assert None != src_id or None != dst_id - super(DBO_load_link_set_by_src_or_dst_id_attributes, self).__init__() + super(DBO_match_link_set_by_src_or_dst_id_attributes, self).__init__() if not src_id: q = "match ()-[r]->({id: {dst_id}}) return r" @@ -243,7 +242,7 @@ class DBO_load_link_set_by_src_or_dst_id_attributes(DB_op): log.debug('loaded id-set: ' + str(data)) return self.parse_single_query_response_data(data) -class DBO_load_link_id_set(DB_op): +class DBO_match_link_id_set(DB_op): def __init__(self, filter_type=None, filter_attr_map={}): """ load an id-set of links @@ -253,7 +252,7 @@ class DBO_load_link_id_set(DB_op): attributes to match link properties against @return: a set of loaded link ids """ - super(DBO_load_link_id_set, self).__init__() + super(DBO_match_link_id_set, self).__init__() q = "match ()-[r{filter_type} {filter_attr}]->() return id(r)" q = cfmt(q, filter_type="" if not filter_type else ":" + filter_type) @@ -280,8 +279,8 @@ class DB_Driver_REST(DB_Driver_Base): # # [!] neo4j seems picky about receiving an additional empty statement list # - data = data = dbu.statement_set_to_REST_form([]) - ret = dbu.post_neo4j(tx_open_url, data) + data = data = db_util.statement_set_to_REST_form([]) + ret = db_util.post_neo4j(tx_open_url, data) tx_commit_url = ret['commit'] op.parse_tx_id(tx_commit_url) @@ -291,10 +290,11 @@ class DB_Driver_REST(DB_Driver_Base): def exex_op_statements(self, op): tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) - statement_set = dbu.statement_set_to_REST_form(op.statement_set) + statement_set = db_util.statement_set_to_REST_form(op.statement_set) try: - ret = dbu.post_neo4j(tx_url, statement_set) + ret = db_util.post_neo4j(tx_url, statement_set) + op._assign_results_errors(ret) self.log_committed_queries(statement_set) return ret except Exception as e: @@ -307,8 +307,8 @@ class DB_Driver_REST(DB_Driver_Base): # # [!] neo4j seems picky about receiving an additional empty statement list # - data = dbu.statement_set_to_REST_form([]) - ret = dbu.post(tx_commit_url, data) + data = db_util.statement_set_to_REST_form([]) + ret = db_util.post(tx_commit_url, data) log.debug('tx-commit: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) @@ -379,5 +379,5 @@ class DB_Controller: @deprecated: use transaction based api """ - # call post and not dbu.post_neo4j to avoid response key errors - dbu.post(self.config.db_base_url + '/db/data/cypher', {"query" : q}) + # call post and not db_util.post_neo4j to avoid response key errors + db_util.post(self.config.db_base_url + '/db/data/cypher', {"query" : q}) diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 120a601a..5169af0d 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -30,9 +30,9 @@ class Cypher_String_Formatter(string.Formatter): def get_field(self, field_name, args, kwargs): # ignore key not found, return bracket wrapped key try: - val=super(Cypher_String_Formatter, self).get_field(field_name, args, kwargs) + val = super(Cypher_String_Formatter, self).get_field(field_name, args, kwargs) except (KeyError, AttributeError): - val="{" + field_name + "}", field_name + val = "{" + field_name + "}", field_name return val def cfmt(fmt_str, *args, **kwargs): @@ -72,7 +72,11 @@ def statement_to_REST_form(query, parameters={}): turn cypher query to neo4j json API format """ assert isinstance(query, basestring) - assert isinstance(parameters, dict) + if isinstance(parameters, list): + for v in parameters: + assert isinstance(v, dict) + else: + assert isinstance(parameters, dict) return {'statement' : query, 'parameters': parameters} @@ -96,13 +100,16 @@ def gen_clause_attr_filter_from_filter_attr_map(filter_attr_map, node_label="n") filter_str = "{{{0}}}".format(', '.join(filter_arr)) return filter_str + def gen_clause_where_from_filter_attr_map(filter_attr_map, node_label="n"): """ convert a filter attribute map to a parameterized Cypher where clause, eg. in: { 'att_foo': [ 'a', 'b' ], 'att_goo': [1,2] } - out: where n.att_foo in {att_foo} and n.att_goo in {att_goo} ... + out: {att_foo: {att_foo}, att_goo: {att_goo}, ...} + + this function will essentially ignore all but the first value in the value list - @param filter_attr_map: may be None or empty + @param filter_attr_map: may be None or empty """ if not filter_attr_map: return "" @@ -194,8 +201,6 @@ def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__type'): ret[v_type].append(v_no_meta) return ret - q = "match (src {id: {src}.id}),(dst {id: {dst}.id}) create (src)-[:%(l_type)s {link_attr}]->(dst)" % {'l_type':l_type} - return (q, q_params_set) def __type_check_link(link): assert link.has_key('__src') -- cgit v1.3.1 From 10af77cc4a76448041b333d8ec793751dc3022a1 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 15 Oct 2014 20:25:00 +0200 Subject: DBO_load_node_set_by_DB_id - simplify query --- src-py/db_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index cb52cc09..8e87d9d8 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -173,7 +173,7 @@ class DBO_load_node_set_by_DB_id(DB_op): @return: loaded node set or an empty set if no match was found """ super(DBO_load_node_set_by_DB_id, self).__init__() - q = "match (n) where id(n) in {id_set} return n" + q = "start n=node({id_set}) return n" self.add_statement(q, { 'id_set': id_set}) def on_completion(self, data): -- cgit v1.3.1 From dcbc18c61d0917008c09c7a15ec57045641a9bce Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 15 Oct 2014 22:12:56 +0200 Subject: DBO_attr_diff_commit & test - todo: parse DB response --- src-py/db_controller.py | 24 ++++++++++++++++++++++++ src-py/test_db_controller.py | 15 +++++++++++++++ 2 files changed, 39 insertions(+) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 8e87d9d8..49ed3b6b 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -123,6 +123,30 @@ class DBO_topo_diff_commit(DB_composed_op): self.add_sub_op(op_n_add) self.add_sub_op(op_l_add) + def on_completion(self, data): + pass + +class DBO_attr_diff_commit(DB_op): + """ + commit a Attr_Diff + """ + def __init__(self, attr_diff): + super(DBO_attr_diff_commit, self).__init__() + + for id_attr, n_attr_diff in attr_diff.items(): + # TODO parameterize multiple attr removal + rm_attr_set = n_attr_diff['attr_remove'] + rm_attr_str = ', '.join(['n.' + attr for attr in rm_attr_set]) + + q = ("match (n {id: {id}}) " + + "set n += {attr_set}" + + "remove " + rm_attr_str + + " return n.id, n") + q_param_set = {'id': id_attr, + 'attr_set': n_attr_diff['attr_write']} + self.add_statement(q, q_param_set) + + def on_completion(self, data): pass class DBO_add_node_set(DB_op): diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 254afb80..66ecfd09 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -193,6 +193,21 @@ class TestDBController(unittest.TestCase): id_set = self.db_ctl.exec_op(dbc.DBO_match_link_set_by_src_or_dst_id_attributes(src_id=n_1_id, dst_id=n_0_id)) self.assertEqual(len(id_set), 1) + def test_attr_diff_commit(self): + # create test node + n_id = rand_id() + topo_diff = Topo_Diff(node_set_add=[{'__type': 'T_test_attr_diff_commit', 'id': n_id, 'attr_rm': 0}]) + op = dbc.DBO_topo_diff_commit(topo_diff) + self.db_ctl.exec_op(op) + + # apply attr_diff + attr_diff = {n_id: {'attr_write': {'attr_0': 0, + 'attr_1': 'a'}, + 'attr_remove': ['attr_rm']}} + op = dbc.DBO_attr_diff_commit(attr_diff) + n_set = self.db_ctl.exec_op(op) + pass + def tearDown(self): pass if __name__ == "__main__": -- cgit v1.3.1 From d3d8bd18bfbf2da267ea30f5625f092d24b21474 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Thu, 16 Oct 2014 16:01:33 +0200 Subject: DB_row, update DBO_XXX iteration loops, test updates --- src-py/db_controller.py | 15 +++++++++------ src-py/neo4j_util.py | 12 ++++++++++-- src-py/test_db_controller.py | 14 ++++++++++---- 3 files changed, 29 insertions(+), 12 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 49ed3b6b..f4bfbb75 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -67,7 +67,8 @@ class DB_op(object): ret = [] for _, _, r_set in self: for row in r_set: - ret.append(row) + for cloumn in row: + ret.append(row) return ret def parse_multi_statement_response_data(self, data): @@ -155,6 +156,7 @@ class DBO_add_node_set(DB_op): DB op: add node set @param node_map: node-type to node-set map + @return: set of new node DB ids """ super(DBO_add_node_set, self).__init__() for q, q_param_set in db_util.gen_query_create_from_node_map(node_map): @@ -164,8 +166,8 @@ class DBO_add_node_set(DB_op): id_set = [] for _, _, r_set in self: for row in r_set: - nid = row # [!] fragile - id_set.append(nid) + for clo in row: + id_set.append(clo) log.debug('node-set added: ids: ' + str(id_set)) return id_set @@ -174,6 +176,7 @@ class DBO_add_link_set(DB_op): def __init__(self, link_map): """ @param link_map: is a link-type to link-set map - see model.link + @return: set of new node DB ids """ super(DBO_add_link_set, self).__init__() for q, q_params in db_util.gen_query_create_from_link_map(link_map): @@ -181,10 +184,10 @@ class DBO_add_link_set(DB_op): def on_completion(self, data): id_set = [] - for s_id, s, r_set in self: + for _, _, r_set in self: for row in r_set: - lid = row # [!] fragile - id_set.append(lid) + for col_val in row: + id_set.append(col_val) log.debug('link-set added: ids: ' + str(id_set)) return id_set diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 5169af0d..8d70ced9 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -7,6 +7,14 @@ import urllib2 import model import string +class DB_row(object): + def __init__(self, data): + self.data = data + + def __iter__(self): + for column_val in self.data: + yield column_val + class DB_result_set(object): def __init__(self, data): self.data = data @@ -14,9 +22,9 @@ class DB_result_set(object): def __iter__(self): for db_row_dict in self.data['data']: # example: dict: {u'row': [{u'title': u'foo'}]} - assert None != db_row_dict['row'][0] + assert None != db_row_dict['row'] - yield db_row_dict['row'][0] + yield DB_row(db_row_dict['row']) class Cypher_String_Formatter(string.Formatter): """ diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 66ecfd09..edfd4a22 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -135,13 +135,19 @@ class TestDBController(unittest.TestCase): n_set = self.db_ctl.exec_op(op) self.assertEqual(len(n_set), 1) - def test_node_DB_id_lifecycle(self): + def test_load_node_set_by_DB_id(self): """ test node DB id life cycle """ - id_set = self.db_ctl.exec_op(dbc.DBO_add_node_set({'Person': [{'name': 'John Doe', 'id': 'jdoe_00'}, - {'name': 'John Doe', 'id': 'jdoe_01'}]})) - n_set = self.db_ctl.exec_op(dbc.DBO_load_node_set_by_DB_id(id_set)) + + # create nodes, get DB ids + op = dbc.DBO_add_node_set({'T_test_load_node_set_by_DB_id': [{'name': 'John Doe'}, + {'name': 'John Doe'}]}) + id_set = self.db_ctl.exec_op(op) + + # match against DB ids + op = dbc.DBO_load_node_set_by_DB_id(id_set) + n_set = self.db_ctl.exec_op(op) self.assertEqual(len(n_set), len(id_set), 'incorrect result size') def test_partial_query_set_execution_success(self): -- cgit v1.3.1 From 3b3d3b5e91dac517627f1a080fffbff9a72b7e1a Mon Sep 17 00:00:00 2001 From: LV-426 Date: Thu, 16 Oct 2014 16:30:22 +0200 Subject: DBO_attr_diff_commit on_completion response processing, test_attr_diff_commit() update --- src-py/db_controller.py | 11 ++++++++--- src-py/test_db_controller.py | 9 +++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index f4bfbb75..2c18427c 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -141,14 +141,19 @@ class DBO_attr_diff_commit(DB_op): q = ("match (n {id: {id}}) " + "set n += {attr_set}" + - "remove " + rm_attr_str + - " return n.id, n") + "remove " + rm_attr_str + " " + + "return n.id, n") q_param_set = {'id': id_attr, 'attr_set': n_attr_diff['attr_write']} self.add_statement(q, q_param_set) def on_completion(self, data): - pass + ret = {} + for _, _, r_set in self: + for row in r_set: + n_id, n = [v for v in row] # we expect a [n_id, n] array + ret[n_id] = n + return ret class DBO_add_node_set(DB_op): def __init__(self, node_map): diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index edfd4a22..c7834749 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -211,8 +211,13 @@ class TestDBController(unittest.TestCase): 'attr_1': 'a'}, 'attr_remove': ['attr_rm']}} op = dbc.DBO_attr_diff_commit(attr_diff) - n_set = self.db_ctl.exec_op(op) - pass + n_map = self.db_ctl.exec_op(op) + self.assertEqual(len(n_map), 1) + n = n_map.get(n_id) + self.assertTrue(None != n) + self.assertTrue(None == n.get('attr_rm')) + self.assertTrue(None != n.get('attr_0')) + self.assertTrue(None != n.get('attr_1')) def tearDown(self): pass -- cgit v1.3.1 From bf3e34d7835aa75c5ff4d131d327143e611c4066 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 20 Oct 2014 17:09:44 +0200 Subject: fix missing ' ' in DBO_attr_diff_commit() query --- src-py/db_controller.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 2c18427c..64e9b5eb 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -138,9 +138,9 @@ class DBO_attr_diff_commit(DB_op): # TODO parameterize multiple attr removal rm_attr_set = n_attr_diff['attr_remove'] rm_attr_str = ', '.join(['n.' + attr for attr in rm_attr_set]) - - q = ("match (n {id: {id}}) " + - "set n += {attr_set}" + + + q = ("match (n {id: {id}}) " + + "set n += {attr_set} " + "remove " + rm_attr_str + " " + "return n.id, n") q_param_set = {'id': id_attr, -- cgit v1.3.1 From 92668e664144b15797f83c2abd61addac8b3c43a Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 20 Oct 2014 19:21:59 +0200 Subject: DBO_attr_diff_commit - augment query generation --- src-py/db_controller.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 64e9b5eb..b282a38b 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -136,15 +136,25 @@ class DBO_attr_diff_commit(DB_op): for id_attr, n_attr_diff in attr_diff.items(): # TODO parameterize multiple attr removal - rm_attr_set = n_attr_diff['attr_remove'] - rm_attr_str = ', '.join(['n.' + attr for attr in rm_attr_set]) - - q = ("match (n {id: {id}}) " + - "set n += {attr_set} " + - "remove " + rm_attr_str + " " + - "return n.id, n") - q_param_set = {'id': id_attr, - 'attr_set': n_attr_diff['attr_write']} + r_attr_set = n_attr_diff['attr_remove'] + w_attr_set = n_attr_diff['attr_write'] + + assert len(r_attr_set) > 0 or len(w_attr_set) > 0 + + q_arr = ["match (n {id: {id}}) ", + "return n.id, n"] + q_param_set = {'id': id_attr} + + if len(r_attr_set) > 0: + stmt_attr_rm = "remove " + ', '.join(['n.' + attr for attr in r_attr_set]) + q_arr.insert(1, stmt_attr_rm) + + if len(w_attr_set) > 0: + stmt_attr_set = "set n += {attr_set}" + q_arr.insert(1, stmt_attr_set) + q_param_set['attr_set'] = w_attr_set + + q = " ".join(q_arr) self.add_statement(q, q_param_set) def on_completion(self, data): -- cgit v1.3.1 From 3ad760c3b77dab1b3039d9ff757aef61a2068b17 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 20 Oct 2014 19:25:23 +0200 Subject: special handling of DB_composed_op execution --- src-py/db_controller.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index b282a38b..632e1dcd 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -86,22 +86,38 @@ class DB_composed_op(DB_op): super(DB_composed_op, self).__init__() self.sub_op_set = [] - def add_statement(self, query, query_params={}): + def __assert_false_statement_access(self): assert False, "composed_op may not contain statements, only sub-ops" + def add_statement(self, query, query_params={}): + self.__assert_false_statement_access() + def add_sub_op(self, op): self.sub_op_set.append(op) def __getattribute__(self, attr): """ - intercept 'statement_set' attr get + intercept 'statement_set' attr get """ if attr == 'statement_set': - # construct a list comprehension composed of all sup_op statements - return [s for s_op in self.sub_op_set for s in s_op.statement_set] + self.__assert_false_statement_access() return object.__getattribute__(self, attr) + def __iter__(self): + """ + iterate over sub_op_set + """ + for s_op in self.sub_op_set: + yield s_op + + def process_result_set(self): + ret = [] + for s_op in self: + s_result_set = s_op.process_result_set() + ret.append(s_result_set) + return ret + class DBO_topo_diff_commit(DB_composed_op): """ commit a @@ -401,16 +417,24 @@ class DB_Controller: """ execute operation within a DB transaction """ + if isinstance(op, DB_composed_op): + # construct a list comprehension composed of all sup_op statements + for s_op in op.sub_op_set: + self.exec_op(s_op) + return op.process_result_set() + try: self.db_driver.begin_tx(op) - ret_tx = self.db_driver.exex_op_statements(op) - ret_commit = self.db_driver.commit_tx(op) - return op.on_completion(ret_tx) + self.db_driver.exec_statement_set(op) + self.db_driver.commit_tx(op) + + return op.process_result_set() except Exception as e: # here we watch for IOExecptions, etc - not db errors # these are returned in the db response itself log.error(e.message) log.error(traceback.print_exc()) + raise e def create_db_op(self, f_work, f_cont): ret = DB_op(f_work, f_cont) -- cgit v1.3.1 From 81cdcfa9cac04031082d93bb9e7bf6c844b7b75b Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 20 Oct 2014 19:25:52 +0200 Subject: misc --- src-py/db_controller.py | 52 ++++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 24 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 632e1dcd..bd085371 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -6,10 +6,15 @@ import re import logging import traceback +from model.graph import Attr_Diff +from model.graph import Topo_Diff + import urllib2 -import neo4j_util as dbu +import neo4j_util as db_util +from neo4j_util import cfmt from neo4j_util import DB_result_set +from neo4j_util import Neo4JException log = logging.getLogger('rhizi') @@ -326,6 +331,28 @@ class DBO_match_link_id_set(DB_op): class DB_Driver_Base(): pass +class DB_Driver_Embedded(DB_Driver_Base): + def __init__(self, db_base_url): + self.tx_base_url = db_base_url + '/db/data/transaction' + + from org.rhizi.db.neo4j.util import EmbeddedNeo4j + self.edb = EmbeddedNeo4j.createDb() + self.edb.createDb() + + def begin_tx(self, op): + pass + + def exec_statement_set(self, op): + s_set = op.statement_set + self.edb.executeCypherQury() + + def commit_tx(self, op): + pass + + def log_committed_queries(self, statement_set): + for sp_dict in statement_set['statements']: + log.debug('\tq: {0}'.format(sp_dict['statement'])) + class DB_Driver_REST(DB_Driver_Base): def __init__(self, db_base_url): self.tx_base_url = db_base_url + '/db/data/transaction' @@ -378,29 +405,6 @@ class DB_Driver_REST(DB_Driver_Base): for sp_dict in statement_set['statements']: log.debug('\tq: {0}'.format(sp_dict['statement'])) -class DB_Driver_Embedded(DB_Driver_Base): - def __init__(self, db_base_url): - self.tx_base_url = db_base_url + '/db/data/transaction' - - from org.rhizi.db.neo4j.util import EmbeddedNeo4j - self.edb = EmbeddedNeo4j.createDb() - self.edb.createDb() - - def begin_tx(self, op): - pass - - def exex_op_statements(self, op): - s_set = op.statement_set - self.edb.executeCypherQury() - - def commit_tx(self, op): - pass - - def log_committed_queries(self, statement_set): - for sp_dict in statement_set['statements']: - log.debug('\tq: {0}'.format(sp_dict['statement'])) - - class DB_Controller: """ neo4j DB controller -- cgit v1.3.1 From 133fb2a63c260a26730d44bf79d87eee360558c5 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 20 Oct 2014 19:27:09 +0200 Subject: DB_composed_op - optimize for empty n_add, l_add sets --- src-py/db_controller.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index bd085371..ddc25a85 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -125,28 +125,29 @@ class DB_composed_op(DB_op): class DBO_topo_diff_commit(DB_composed_op): """ - commit a + commit a Topo_Diff """ def __init__(self, topo_diff): super(DBO_topo_diff_commit, self).__init__() # TODO rm link set # TODO rm node set - assert not topo_diff.node_set_rm, 'unsupported' - assert not topo_diff.link_set_rm, 'unsupported' + assert 0 == len(topo_diff.node_set_rm), 'unsupported' + assert 0 == len(topo_diff.link_set_rm), 'unsupported' n_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.node_set_add) l_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.link_set_add) - op_n_add = DBO_add_node_set(n_add_map) - op_l_add = DBO_add_link_set(l_add_map) - + # # [!] order critical - self.add_sub_op(op_n_add) - self.add_sub_op(op_l_add) - - def on_completion(self, data): - pass + # + if len(n_add_map) > 0: + op_n_add = DBO_add_node_set(n_add_map) + self.add_sub_op(op_n_add) + + if len(l_add_map) > 0: + op_l_add = DBO_add_link_set(l_add_map) + self.add_sub_op(op_l_add) class DBO_attr_diff_commit(DB_op): """ -- cgit v1.3.1 From 00321278f858ec36fa4c799363bbcc508164f13c Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 20 Oct 2014 19:30:04 +0200 Subject: query result processing: - bubble up common processing - rename on_completion --- src-py/db_controller.py | 62 +++++++++++++++---------------------------------- 1 file changed, 19 insertions(+), 43 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index ddc25a85..0ef70f3b 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -25,7 +25,7 @@ class DB_op(object): def __init__(self): self.statement_set = [] - self.result_set = None + self.result_set = [] self.error_set = None self.tx_id = None self.tx_commit_url = None # cached from response to tx begin @@ -54,17 +54,18 @@ class DB_op(object): TODO: handle partial iteration due to error_set being non-empty """ i = 0 - if self.result_set: - for s in self.statement_set: - rs = DB_result_set(self.result_set[i]) - yield (i, s, rs) - i = i + 1 - else: - for s in self.statement_set: - yield (i, s, None) - i = i + 1 + r_set_len = len(self.result_set) + for s in self.statement_set: + r_set = None # row-set + if i < r_set_len: # support partial result recovery + r_set = DB_result_set(self.result_set[i]) + yield (i, s, r_set) + i = i + 1 - def parse_single_query_response_data(self, data): + def parse_multi_statement_response_data(self, data): + pass + + def process_result_set(self): """ DB op can issue complex sets of quries all at once - this helper method assists in parsing response data from a single query. @@ -72,19 +73,10 @@ class DB_op(object): ret = [] for _, _, r_set in self: for row in r_set: - for cloumn in row: - ret.append(row) + for col in row: + ret.append(col) return ret - def parse_multi_statement_response_data(self, data): - pass - - def on_completion(self, data): - pass - - def _assign_results_errors(self, data): - self.result_set = data['results'] - self.error_set = data['errors'] class DB_composed_op(DB_op): def __init__(self): @@ -179,7 +171,7 @@ class DBO_attr_diff_commit(DB_op): q = " ".join(q_arr) self.add_statement(q, q_param_set) - def on_completion(self, data): + def process_result_set(self): ret = {} for _, _, r_set in self: for row in r_set: @@ -199,10 +191,10 @@ class DBO_add_node_set(DB_op): for q, q_param_set in db_util.gen_query_create_from_node_map(node_map): self.add_statement(q, q_param_set) - def on_completion(self, data): + def process_result_set(self): id_set = [] - for _, _, r_set in self: - for row in r_set: + for _, _, row_set in self: + for row in row_set: for clo in row: id_set.append(clo) @@ -219,7 +211,7 @@ class DBO_add_link_set(DB_op): for q, q_params in db_util.gen_query_create_from_link_map(link_map): self.add_statement(q, q_params) - def on_completion(self, data): + def process_result_set(self): id_set = [] for _, _, r_set in self: for row in r_set: @@ -240,10 +232,6 @@ class DBO_load_node_set_by_DB_id(DB_op): q = "start n=node({id_set}) return n" self.add_statement(q, { 'id_set': id_set}) - def on_completion(self, data): - log.debug('loaded node set: ' + str(data)) - return self.parse_single_query_response_data(data) - class DBO_match_node_id_set(DB_op): def __init__(self, filter_type=None, filter_attr_map=None): @@ -265,10 +253,6 @@ class DBO_match_node_id_set(DB_op): self.add_statement(q, q_params) - def on_completion(self, data): - log.debug('loaded id-set: ' + str(data)) - return self.parse_single_query_response_data(data) - class DBO_match_node_set_by_id_attribute(DBO_match_node_id_set): def __init__(self, id_set): """ @@ -302,10 +286,6 @@ class DBO_match_link_set_by_src_or_dst_id_attributes(DB_op): self.add_statement(q, q_params) - def on_completion(self, data): - log.debug('loaded id-set: ' + str(data)) - return self.parse_single_query_response_data(data) - class DBO_match_link_id_set(DB_op): def __init__(self, filter_type=None, filter_attr_map={}): """ @@ -325,10 +305,6 @@ class DBO_match_link_id_set(DB_op): self.add_statement(q, q_params) - def on_completion(self, data): - log.debug('loaded id-set: ' + str(data)) - return self.parse_single_query_response_data(data) - class DB_Driver_Base(): pass -- cgit v1.3.1 From f821ead8d1697702917817ce916e50f1c9e0cc3c Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 20 Oct 2014 19:30:34 +0200 Subject: DBO_match_node_id_set query generation --- src-py/db_controller.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 0ef70f3b..875eb0bc 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -234,22 +234,23 @@ class DBO_load_node_set_by_DB_id(DB_op): class DBO_match_node_id_set(DB_op): - def __init__(self, filter_type=None, filter_attr_map=None): + def __init__(self, filter_type=None, filter_attr_map={}): """ - load a set of nodes according to filter_attr_map + match a set of nodes by type / attr_map + @param filter_type: node type filter @param filter_attr_map: is a filter_key to filter_value_set map of - attributes to match against, eg.: + possible attributes to match against, eg.: { 'id':[0,1], 'color: ['red','blue'] } - @param filter_type: node type filter - @return: loaded node set or an empty set if no match was found + @return: a set of node DB id's """ + super(DBO_match_node_id_set, self).__init__() - filter_str = dbu.where_clause_from_filter_attr_map() + q = "match (n{filter_type}) {where_clause} return id(n)" + q = cfmt(q, filter_type="" if not filter_type else ":" + filter_type) + q = cfmt(q, where_clause=db_util.gen_clause_where_from_filter_attr_map(filter_attr_map)) - super(DBO_load_node_set, self).__init__() - q = "match (n) {0} return n".format(filter_str) - self.add_statement(q, params=filter_attr_map) + q_params = filter_attr_map self.add_statement(q, q_params) -- cgit v1.3.1 From 957076abcebb5dc898852b6c58733fa0b408b008 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 20 Oct 2014 19:31:06 +0200 Subject: Neo4JException --- src-py/db_controller.py | 14 ++++++++++---- src-py/neo4j_util.py | 7 +++++++ 2 files changed, 17 insertions(+), 4 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 875eb0bc..7cd1af7b 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -351,15 +351,21 @@ class DB_Driver_REST(DB_Driver_Base): except Exception as e: raise Exception('failed to open transaction:' + e.message) - def exex_op_statements(self, op): + def exec_statement_set(self, op): + tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) statement_set = db_util.statement_set_to_REST_form(op.statement_set) try: - ret = db_util.post_neo4j(tx_url, statement_set) - op._assign_results_errors(ret) + post_ret = db_util.post_neo4j(tx_url, statement_set) + op.result_set = post_ret['results'] + op.error_set = post_ret['errors'] + if 0 != len(op.error_set): + raise Neo4JException(op.error_set) + self.log_committed_queries(statement_set) - return ret + except Neo4JException as e: + raise e except Exception as e: raise Exception('failed exec op statements: err: {0}, url: {1}'.format(e.message, tx_url)) diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 8d70ced9..5b7cfee7 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -7,6 +7,13 @@ import urllib2 import model import string +class Neo4JException(Exception): + def __init__(self, error_set): + self.error_set = error_set + + def __str__(self): + return 'neo4j error set: ' + str(self.error_set) + class DB_row(object): def __init__(self, data): self.data = data -- cgit v1.3.1 From 8d35498c06d3e832bdbcad2916efe2df9b23492d Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 21 Oct 2014 19:56:58 +0200 Subject: introducing the Link_Ptr concept - able to fuzzy point at a link by src_id, dst_id or both --- src-py/db_controller.py | 40 +++++++++++++++++++++++++--------------- src-py/model/model.py | 25 +++++++++++++++++++++++-- src-py/test_db_controller.py | 29 +++++++++++++++++++---------- 3 files changed, 67 insertions(+), 27 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 7cd1af7b..ceb5267c 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -264,28 +264,38 @@ class DBO_match_node_set_by_id_attribute(DBO_match_node_id_set): super(DBO_match_node_set_by_id_attribute, self).__init__(filter_attr_map={'id': id_set}) -class DBO_match_link_set_by_src_or_dst_id_attributes(DB_op): - def __init__(self, src_id=None, dst_id=None): +class DBO_load_link_set(DB_op): + def __init__(self, link_ptr_set): """ - match a set of links by source/target node id attributes + match a set of sets of links by source/target node id attributes + This class should be instantiated through a static factory function + + @link_ptr_set link pointer set @return: a set of loaded links """ - assert None != src_id or None != dst_id + super(DBO_load_link_set, self).__init__() + + for l_ptr in link_ptr_set: + if not l_ptr.src_id: + q = "match ()-[r]->({id: {dst_id}}) return r" + q_params = {'dst_id': l_ptr.dst_id} + elif not l_ptr.dst_id: + q = "match ({id: {src_id}})-[r]->() return r" + q_params = {'src_id': l_ptr.src_id} + else: + q = "match ({id: {src_id}})-[r]->({id: {dst_id}}) return r" + q_params = {'src_id': l_ptr.src_id, 'dst_id': l_ptr.dst_id} - super(DBO_match_link_set_by_src_or_dst_id_attributes, self).__init__() + self.add_statement(q, q_params) - if not src_id: - q = "match ()-[r]->({id: {dst_id}}) return r" - q_params = {'dst_id': dst_id} - elif not dst_id: - q = "match ({id: {src_id}})-[r]->() return r" - q_params = {'src_id': src_id} - else: - q = "match ({id: {src_id}})-[r]->({id: {dst_id}}) return r" - q_params = {'src_id': src_id, 'dst_id': dst_id} + @staticmethod + def init_from_link_ptr(l_ptr): + return DBO_load_link_set([l_ptr]) - self.add_statement(q, q_params) + @staticmethod + def init_from_link_ptr_set(l_ptr_set): + return DBO_load_link_set(l_ptr_set) class DBO_match_link_id_set(DB_op): def __init__(self, filter_type=None, filter_attr_map={}): diff --git a/src-py/model/model.py b/src-py/model/model.py index 9ee61397..701074ab 100644 --- a/src-py/model/model.py +++ b/src-py/model/model.py @@ -1,4 +1,4 @@ -class link(): +class Link(): """ documentation anchor - this class currently carries no implementation and only acts as a documentation anchor @@ -6,4 +6,25 @@ class link(): link['__src'] - meta attribute for link source link['__dst'] - meta attribute for link destination """ - pass + + class Link_Ptr(dict): + def __init__(self, src_id=None, dst_id=None): + assert None != src_id or None != dst_id + + self['__src'] = src_id + self['__dst'] = dst_id + + @property + def src_id(self): + return self['__src'] + + @property + def dst_id(self): + return self['__dst'] + + @staticmethod + def link_ptr(src_id=None, dst_id=None): + """ + init from src_id or dst_id attributes - at least one must be provided + """ + return Link.Link_Ptr(src_id, dst_id) diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 4af4cabc..cdc0c5e9 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -122,19 +122,28 @@ class TestDBController(unittest.TestCase): id_set = self.db_ctl.exec_op(op) self.assertEqual(len(id_set), 0) - def test_match_link_set_by_src_or_dst_id_attributes(self): - op = dbc.DBO_match_link_set_by_src_or_dst_id_attributes(src_id='person_00', dst_id='skill_00') - n_set = self.db_ctl.exec_op(op) - self.assertEqual(len(n_set), 1) + def test_load_link_set(self): + l_ptr = Link.link_ptr(src_id='person_00', dst_id='skill_00') + op = dbc.DBO_load_link_set.init_from_link_ptr(l_ptr) + l_set = self.db_ctl.exec_op(op) + self.assertEqual(len(l_set), 1) - op = dbc.DBO_match_link_set_by_src_or_dst_id_attributes(src_id='person_00') - n_set = self.db_ctl.exec_op(op) - self.assertEqual(len(n_set), 2) + l_ptr = Link.link_ptr(src_id='person_00') + op = dbc.DBO_load_link_set.init_from_link_ptr(l_ptr) + l_set = self.db_ctl.exec_op(op) + self.assertEqual(len(l_set), 2) - op = dbc.DBO_match_link_set_by_src_or_dst_id_attributes(dst_id='skill_00') - n_set = self.db_ctl.exec_op(op) - self.assertEqual(len(n_set), 1) + l_ptr = Link.link_ptr(dst_id='skill_00') + op = dbc.DBO_load_link_set.init_from_link_ptr(l_ptr) + l_set = self.db_ctl.exec_op(op) + self.assertEqual(len(l_set), 1) + # load sets + l_ptr_set = [Link.link_ptr(s,d) for (s,d) in [('person_00', 'skill_00'), ('person_00', 'skill_01')]] + op = dbc.DBO_load_link_set.init_from_link_ptr_set(l_ptr_set) + l_set = self.db_ctl.exec_op(op) + self.assertEqual(len(l_set), 2) + def test_load_node_set_by_DB_id(self): """ test node DB id life cycle -- cgit v1.3.1 From a48c2f33fe91cfb2645013b0174e7ce0e94a04b9 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 21 Oct 2014 19:57:38 +0200 Subject: mv DB_Driver classes to their own file --- src-py/db_controller.py | 83 -------------------------------------------- src-py/db_driver.py | 91 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 83 deletions(-) create mode 100644 src-py/db_driver.py (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index ceb5267c..84c0743e 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -316,89 +316,6 @@ class DBO_match_link_id_set(DB_op): self.add_statement(q, q_params) -class DB_Driver_Base(): - pass - -class DB_Driver_Embedded(DB_Driver_Base): - def __init__(self, db_base_url): - self.tx_base_url = db_base_url + '/db/data/transaction' - - from org.rhizi.db.neo4j.util import EmbeddedNeo4j - self.edb = EmbeddedNeo4j.createDb() - self.edb.createDb() - - def begin_tx(self, op): - pass - - def exec_statement_set(self, op): - s_set = op.statement_set - self.edb.executeCypherQury() - - def commit_tx(self, op): - pass - - def log_committed_queries(self, statement_set): - for sp_dict in statement_set['statements']: - log.debug('\tq: {0}'.format(sp_dict['statement'])) - -class DB_Driver_REST(DB_Driver_Base): - def __init__(self, db_base_url): - self.tx_base_url = db_base_url + '/db/data/transaction' - - def begin_tx(self, op): - tx_open_url = self.tx_base_url - - try: - # - # [!] neo4j seems picky about receiving an additional empty statement list - # - data = data = db_util.statement_set_to_REST_form([]) - ret = db_util.post_neo4j(tx_open_url, data) - tx_commit_url = ret['commit'] - op.parse_tx_id(tx_commit_url) - - log.debug('tx-open: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) - except Exception as e: - raise Exception('failed to open transaction:' + e.message) - - def exec_statement_set(self, op): - - tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) - statement_set = db_util.statement_set_to_REST_form(op.statement_set) - - try: - post_ret = db_util.post_neo4j(tx_url, statement_set) - op.result_set = post_ret['results'] - op.error_set = post_ret['errors'] - if 0 != len(op.error_set): - raise Neo4JException(op.error_set) - - self.log_committed_queries(statement_set) - except Neo4JException as e: - raise e - except Exception as e: - raise Exception('failed exec op statements: err: {0}, url: {1}'.format(e.message, tx_url)) - - def commit_tx(self, op): - tx_commit_url = "{0}/{1}/commit".format(self.tx_base_url, op.tx_id) - - try: - # - # [!] neo4j seems picky about receiving an additional empty statement list - # - data = db_util.statement_set_to_REST_form([]) - ret = db_util.post(tx_commit_url, data) - - log.debug('tx-commit: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) - - return ret - except Exception as e: - raise Exception('failed to commit transaction:' + e.message) - - def log_committed_queries(self, statement_set): - for sp_dict in statement_set['statements']: - log.debug('\tq: {0}'.format(sp_dict['statement'])) - class DB_Controller: """ neo4j DB controller diff --git a/src-py/db_driver.py b/src-py/db_driver.py new file mode 100644 index 00000000..11eace2c --- /dev/null +++ b/src-py/db_driver.py @@ -0,0 +1,91 @@ +import logging + +from neo4j_util import Neo4JException +import neo4j_util as db_util + + +log = logging.getLogger('rhizi') + +class DB_Driver_Base(): + pass + +class DB_Driver_Embedded(DB_Driver_Base): + def __init__(self, db_base_url): + self.tx_base_url = db_base_url + '/db/data/transaction' + + from org.rhizi.db.neo4j.util import EmbeddedNeo4j + self.edb = EmbeddedNeo4j.createDb() + self.edb.createDb() + + def begin_tx(self, op): + pass + + def exec_statement_set(self, op): + s_set = op.statement_set + self.edb.executeCypherQury() + + def commit_tx(self, op): + pass + + def log_committed_queries(self, statement_set): + for sp_dict in statement_set['statements']: + log.debug('\tq: {0}'.format(sp_dict['statement'])) + +class DB_Driver_REST(DB_Driver_Base): + def __init__(self, db_base_url): + self.tx_base_url = db_base_url + '/db/data/transaction' + + def begin_tx(self, op): + tx_open_url = self.tx_base_url + + try: + # + # [!] neo4j seems picky about receiving an additional empty statement list + # + data = data = db_util.statement_set_to_REST_form([]) + ret = db_util.post_neo4j(tx_open_url, data) + tx_commit_url = ret['commit'] + op.parse_tx_id(tx_commit_url) + + log.debug('tx-open: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) + except Exception as e: + raise Exception('failed to open transaction:' + e.message) + + def exec_statement_set(self, op): + + tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) + statement_set = db_util.statement_set_to_REST_form(op.statement_set) + + try: + post_ret = db_util.post_neo4j(tx_url, statement_set) + op.result_set = post_ret['results'] + op.error_set = post_ret['errors'] + if 0 != len(op.error_set): + raise Neo4JException(op.error_set) + + self.log_committed_queries(statement_set) + except Neo4JException as e: + raise e + except Exception as e: + raise Exception('failed exec op statements: err: {0}, url: {1}'.format(e.message, tx_url)) + + def commit_tx(self, op): + tx_commit_url = "{0}/{1}/commit".format(self.tx_base_url, op.tx_id) + + try: + # + # [!] neo4j seems picky about receiving an additional empty statement list + # + data = db_util.statement_set_to_REST_form([]) + ret = db_util.post(tx_commit_url, data) + + log.debug('tx-commit: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) + + return ret + except Exception as e: + raise Exception('failed to commit transaction:' + e.message) + + def log_committed_queries(self, statement_set): + for sp_dict in statement_set['statements']: + log.debug('\tq: {0}'.format(sp_dict['statement'])) + -- cgit v1.3.1 From 9b89717dfc286af0127108b221630f9c1d054971 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 27 Oct 2014 12:31:48 +0200 Subject: DBO_rm_node_set & test --- src-py/db_controller.py | 19 +++++++++++++++++++ src-py/test_db_controller.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 84c0743e..c73a6882 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -316,6 +316,25 @@ class DBO_match_link_id_set(DB_op): self.add_statement(q, q_params) +class DBO_rm_node_set(DB_op): + def __init__(self, id_set, rm_links=False): + super(DBO_rm_node_set, self).__init__() + + if rm_links: + q_arr = ['match (n)', + 'where n.id in ' + str(id_set), + 'optional match (n)-[r]-()', + 'delete n,r' + ] + else: + q_arr = ['match (n)', + 'where n.id in ' + str(id_set), + 'delete n' + ] + + q = ' '.join(q_arr) # TODO: use id param upon neo4j support: q_params = {'id_set': id_set} + self.add_statement(q) + class DB_Controller: """ neo4j DB controller diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 54952df1..4b17ad3c 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -259,6 +259,37 @@ class TestDBController(unittest.TestCase): op = dbc.DBO_attr_diff_commit(attr_diff) n_map = self.db_ctl.exec_op(op) + def test_rm_node_set(self): + n_0_id = rand_id() + n_1_id = rand_id() + n_2_id = rand_id() + n_3_id = rand_id() + n_T = 'T_test_rm_node_set' + + n_set = [{'__type': n_T, 'id': n_0_id }, + {'__type': n_T, 'id': n_1_id }, + {'__type': n_T, 'id': n_2_id }, + {'__type': n_T, 'id': n_3_id }] + l_set = [{'__type': n_T, '__src': n_2_id, '__dst': n_2_id}, + {'__type': n_T, '__src': n_2_id, '__dst': n_3_id}] + + topo_diff = Topo_Diff(node_set_add=n_set, + link_set_add=l_set) + + op = dbc.DBO_topo_diff_commit(topo_diff) + self.db_ctl.exec_op(op) + + op = dbc.DBO_rm_node_set([n_0_id, n_1_id]) + self.db_ctl.exec_op(op) + + op = dbc.DBO_rm_node_set([n_2_id, n_3_id], rm_links=True) + self.db_ctl.exec_op(op) + + # assert all deleted + op = dbc.DBO_match_node_id_set(filter_type=n_T) + id_set = self.db_ctl.exec_op(op) + self.assertEqual(len(id_set), 0) + def tearDown(self): pass if __name__ == "__main__": -- cgit v1.3.1 From 6629438b2f1a9d54d152281342d4d7e0641b7254 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 27 Oct 2014 12:33:02 +0200 Subject: support node removals in DBO_topo_diff_commit --- src-py/db_controller.py | 18 +++++++++++++----- src-py/test_db_controller.py | 27 ++++++++++++++++++++------- 2 files changed, 33 insertions(+), 12 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index c73a6882..aa1e8450 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -124,22 +124,30 @@ class DBO_topo_diff_commit(DB_composed_op): # TODO rm link set # TODO rm node set - assert 0 == len(topo_diff.node_set_rm), 'unsupported' assert 0 == len(topo_diff.link_set_rm), 'unsupported' n_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.node_set_add) l_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.link_set_add) + l_rm_set = [] + n_rm_set = topo_diff.node_set_rm # # [!] order critical # if len(n_add_map) > 0: - op_n_add = DBO_add_node_set(n_add_map) - self.add_sub_op(op_n_add) + op = DBO_add_node_set(n_add_map) + self.add_sub_op(op) if len(l_add_map) > 0: - op_l_add = DBO_add_link_set(l_add_map) - self.add_sub_op(op_l_add) + op = DBO_add_link_set(l_add_map) + self.add_sub_op(op) + + if len(l_rm_set) > 0: + pass + + if len(n_rm_set) > 0: + op = DBO_rm_node_set(n_rm_set) + self.add_sub_op(op) class DBO_attr_diff_commit(DB_op): """ diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 4b17ad3c..002777d7 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -200,12 +200,13 @@ class TestDBController(unittest.TestCase): n_0_id = rand_id() n_1_id = rand_id() n_2_id = rand_id() + n_T = 'T_test_topo_diff_commit' - n_set = [{'__type': 'T_test_topo_diff_commit', 'id': n_0_id }, - {'__type': 'T_test_topo_diff_commit', 'id': n_1_id }, - {'__type': 'T_test_topo_diff_commit', 'id': n_2_id }] - l_set = [{'__type': 'T_test_topo_diff_commit', '__src': n_0_id, '__dst': n_1_id}, - {'__type': 'T_test_topo_diff_commit', '__src': n_1_id, '__dst': n_0_id}] + n_set = [{'__type': n_T, 'id': n_0_id }, + {'__type': n_T, 'id': n_1_id }, + {'__type': n_T, 'id': n_2_id }] + l_set = [{'__type': n_T, '__src': n_0_id, '__dst': n_1_id}, + {'__type': n_T, '__src': n_1_id, '__dst': n_0_id}] topo_diff = Topo_Diff(node_set_add=n_set, link_set_add=l_set) @@ -218,11 +219,23 @@ class TestDBController(unittest.TestCase): id_set = self.db_ctl.exec_op(dbc.DBO_match_node_set_by_id_attribute([n_0_id, n_1_id])) self.assertEqual(len(id_set), 2) - id_set = self.db_ctl.exec_op(dbc.DBO_load_link_set_by_src_or_dst_id_attributes(src_id=n_0_id, dst_id=n_1_id)) + + l_ptr = Link.link_ptr(src_id=n_0_id, dst_id=n_1_id) + id_set = self.db_ctl.exec_op(dbc.DBO_load_link_set.init_from_link_ptr(l_ptr)) self.assertEqual(len(id_set), 1) - id_set = self.db_ctl.exec_op(dbc.DBO_load_link_set_by_src_or_dst_id_attributes(src_id=n_1_id, dst_id=n_0_id)) + + l_ptr = Link.link_ptr(src_id=n_1_id, dst_id=n_0_id) + id_set = self.db_ctl.exec_op(dbc.DBO_load_link_set.init_from_link_ptr(l_ptr)) self.assertEqual(len(id_set), 1) + id_set_rm=[n_2_id] + topo_diff = Topo_Diff(node_set_rm=id_set_rm) + op = dbc.DBO_topo_diff_commit(topo_diff) + self.db_ctl.exec_op(op) + op = dbc.DBO_match_node_set_by_id_attribute(id_set_rm) + id_set = self.db_ctl.exec_op(op) + self.assertEqual(len(id_set), 0) + def test_attr_diff_commit(self): # create test node n_id = rand_id() -- cgit v1.3.1 From 69d70685cfdea0ea6b6083d36f6fefd0be0741fe Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 27 Oct 2014 12:44:49 +0200 Subject: misc --- src-py/db_controller.py | 18 +++++++++--------- src-py/rhizi_api.py | 24 ++++++++++++++++++++---- 2 files changed, 29 insertions(+), 13 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index aa1e8450..f9656212 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -1,20 +1,19 @@ #!/usr/bin/python -import os import json -import re import logging +import os +import re import traceback +import urllib2 +from db_driver import DB_Driver_REST, DB_Driver_Base from model.graph import Attr_Diff from model.graph import Topo_Diff - -import urllib2 - -import neo4j_util as db_util -from neo4j_util import cfmt from neo4j_util import DB_result_set -from neo4j_util import Neo4JException +from neo4j_util import cfmt +import neo4j_util as db_util +from model.model import Link log = logging.getLogger('rhizi') @@ -183,7 +182,7 @@ class DBO_attr_diff_commit(DB_op): ret = {} for _, _, r_set in self: for row in r_set: - n_id, n = [v for v in row] # we expect a [n_id, n] array + n_id, n = [v for v in row] # we expect a [n_id, n] array ret[n_id] = n return ret @@ -234,6 +233,7 @@ class DBO_load_node_set_by_DB_id(DB_op): """ load a set of nodes whose DB id is in id_set + @param id_set: DB node id set @return: loaded node set or an empty set if no match was found """ super(DBO_load_node_set_by_DB_id, self).__init__() diff --git a/src-py/rhizi_api.py b/src-py/rhizi_api.py index c22df635..7a6a44e9 100644 --- a/src-py/rhizi_api.py +++ b/src-py/rhizi_api.py @@ -48,26 +48,42 @@ def __common_resp_handle(data=None, error=None): ret_data = __response_wrap(data, error) resp = jsonify(ret_data) + resp.headers['Access-Control-Allow-Origin'] = '*' + # more response processing return resp +def __common_exec(op, on_success=__common_resp_handle): + try: + op_ret = db_ctl.exec_op(op) + return on_success(op_ret) + except Exception as e: + return __common_resp_handle('exception raised: add_node_set') -@webapp.route("/load/node-set", methods=['POST']) +@webapp.route("/load/node-set-by-id", methods=['POST']) def load_node_set_by_id_attr(): """ + load node-set by ID attribute + @param id_set: list of node ids to match id attribute against - @return: a list containing a single node whose id attribute matches 'id' or + @return: a list of nodes whose id attribute matches 'id' or an empty list if the requested node is not found @raise exception: on error """ - id_set = request.get_json()['id_set'] + req_json = request.get_json() + id_set = req_json['id_set'] + __sanitize_input(id_set) return __load_node_set_by_id_attr_common(id_set) def __load_node_set_by_id_attr_common(id_set): - op = dbc.DBO_load_node_set_by_id_attribute(id_set) + """ + @param f_k: optional attribute filter key + @param f_vset: possible key values to match against + """ + op = dbc.DBO_match_node_set_by_id_attribute(id_set=id_set) try: n_set = db_ctl.exec_op(op) return __common_resp_handle(data=n_set) -- cgit v1.3.1 From a4ec4c13f3dfaebe3e1b727108e7bc7e05b7fe2d Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 28 Oct 2014 00:31:15 +0200 Subject: DBO_cypher_query --- src-py/db_controller.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index f9656212..f43c9673 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -114,6 +114,14 @@ class DB_composed_op(DB_op): ret.append(s_result_set) return ret +class DBO_cypher_query(DB_op): + """ + freeform cypher query + """ + def __init__(self, q, q_params={}): + super(DBO_cypher_query, self).__init__() + self.add_statement(q, q_params) + class DBO_topo_diff_commit(DB_composed_op): """ commit a Topo_Diff @@ -384,8 +392,13 @@ class DB_Controller: def exec_cypher_query(self, q): """ - @deprecated: use transaction based api + @deprecated: use DBO_cypher_query """ # call post and not db_util.post_neo4j to avoid response key errors - db_util.post(self.config.db_base_url + '/db/data/cypher', {"query" : q}) + try: + db_util.post(self.config.db_base_url + '/db/data/cypher', {"query" : q}) + except Exception as e: + log.error(e.message) + log.error(traceback.print_exc()) + raise e -- cgit v1.3.1 From 89915ecebb2c3f8994ce3b727050ce35714acd78 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 28 Oct 2014 00:31:55 +0200 Subject: initial DBO_rz_clone op --- src-py/db_controller.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index f43c9673..a86de1a8 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -351,6 +351,51 @@ class DBO_rm_node_set(DB_op): q = ' '.join(q_arr) # TODO: use id param upon neo4j support: q_params = {'id_set': id_set} self.add_statement(q) +class DBO_rz_clone(DB_op): + def __init__(self, filter_label=None, limit=128): + """ + clone rhizi + + @return: a dict: {'node_set': n_set, + 'link_set': l_set } + where l_set is a list of (src.id, dst.id, link) tuples + """ + super(DBO_rz_clone, self).__init__() + + self.limit = limit + self.skip = 0 + + q_arr = ['match (n)' if not filter_label else 'match (n:%s)' % (filter_label), + 'optional match (n)-[r]->(m)', + 'with n,r,m', + 'order by n.id', + 'skip %d' % (self.skip), + 'limit %d' % (self.limit), + 'return n,collect([n.id, m.id, r])'] + + q = ' '.join(q_arr) + self.add_statement(q) + + def process_result_set(self): + ret_n_set = [] + ret_l_set = [] + for _, _, row_set in self: + for row in row_set: + itr = iter(row) + n = itr.next() + + l_set = itr.next() + for l in l_set: + assert 3 == len(l) # (n.id, m.id, r) tuples + if None == l[1]: + # as link matching is optional, collect may yield empty sets + continue + ret_l_set.append(l) + ret_n_set.append(n) + + return {'node_set': ret_n_set, + 'link_set': ret_l_set } + class DB_Controller: """ neo4j DB controller -- cgit v1.3.1 From 414afa4b510d49e90a4a776bdeb4c9c066a64922 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Tue, 28 Oct 2014 00:35:48 +0200 Subject: follow neo4j semantics: rename filter_type to filter_label --- src-py/db_controller.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index a86de1a8..a924a164 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -21,7 +21,6 @@ class DB_op(object): """ tx wrapped DB operation possibly composing multiple DB queries """ - def __init__(self): self.statement_set = [] self.result_set = [] @@ -76,7 +75,6 @@ class DB_op(object): ret.append(col) return ret - class DB_composed_op(DB_op): def __init__(self): super(DB_composed_op, self).__init__() @@ -250,11 +248,11 @@ class DBO_load_node_set_by_DB_id(DB_op): class DBO_match_node_id_set(DB_op): - def __init__(self, filter_type=None, filter_attr_map={}): + def __init__(self, filter_label=None, filter_attr_map={}): """ match a set of nodes by type / attr_map - @param filter_type: node type filter + @param filter_label: node type filter @param filter_attr_map: is a filter_key to filter_value_set map of possible attributes to match against, eg.: { 'id':[0,1], 'color: ['red','blue'] } @@ -262,8 +260,8 @@ class DBO_match_node_id_set(DB_op): """ super(DBO_match_node_id_set, self).__init__() - q = "match (n{filter_type}) {where_clause} return id(n)" - q = cfmt(q, filter_type="" if not filter_type else ":" + filter_type) + q = "match (n{filter_label}) {where_clause} return id(n)" + q = cfmt(q, filter_label="" if not filter_label else ":" + filter_label) q = cfmt(q, where_clause=db_util.gen_clause_where_from_filter_attr_map(filter_attr_map)) q_params = filter_attr_map @@ -291,7 +289,7 @@ class DBO_load_link_set(DB_op): @return: a set of loaded links """ super(DBO_load_link_set, self).__init__() - + for l_ptr in link_ptr_set: if not l_ptr.src_id: q = "match ()-[r]->({id: {dst_id}}) return r" @@ -314,19 +312,19 @@ class DBO_load_link_set(DB_op): return DBO_load_link_set(l_ptr_set) class DBO_match_link_id_set(DB_op): - def __init__(self, filter_type=None, filter_attr_map={}): + def __init__(self, filter_label=None, filter_attr_map={}): """ load an id-set of links - @param filter_type: link type filter + @param filter_label: link type filter @param filter_attr_map: is a filter_key to filter_value_set map of attributes to match link properties against @return: a set of loaded link ids """ super(DBO_match_link_id_set, self).__init__() - q = "match ()-[r{filter_type} {filter_attr}]->() return id(r)" - q = cfmt(q, filter_type="" if not filter_type else ":" + filter_type) + q = "match ()-[r{filter_label} {filter_attr}]->() return id(r)" + q = cfmt(q, filter_label="" if not filter_label else ":" + filter_label) q = cfmt(q, filter_attr=db_util.gen_clause_attr_filter_from_filter_attr_map(filter_attr_map)) q_params = {k: v[0] for (k, v) in filter_attr_map.items()} # pass on only first value from each value set @@ -334,6 +332,9 @@ class DBO_match_link_id_set(DB_op): class DBO_rm_node_set(DB_op): def __init__(self, id_set, rm_links=False): + """ + remove node set + """ super(DBO_rm_node_set, self).__init__() if rm_links: -- cgit v1.3.1 From 223adad23867e19ba4d43e14104d40309c54280a Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 9 Nov 2014 12:07:18 +0200 Subject: unify logging --- src-py/db_controller.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index a924a164..40038ed8 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -211,7 +211,6 @@ class DBO_add_node_set(DB_op): for clo in row: id_set.append(clo) - log.debug('node-set added: ids: ' + str(id_set)) return id_set class DBO_add_link_set(DB_op): @@ -231,7 +230,6 @@ class DBO_add_link_set(DB_op): for col_val in row: id_set.append(col_val) - log.debug('link-set added: ids: ' + str(id_set)) return id_set class DBO_load_node_set_by_DB_id(DB_op): @@ -424,7 +422,10 @@ class DB_Controller: self.db_driver.exec_statement_set(op) self.db_driver.commit_tx(op) - return op.process_result_set() + ret = op.process_result_set() + + log.debug('exec_op:' + op.name + ': return value: ' + str(ret)) + return ret except Exception as e: # here we watch for IOExecptions, etc - not db errors # these are returned in the db response itself -- cgit v1.3.1 From 85feae0a74a97ad7bc04e13b511601e14be5f234 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 9 Nov 2014 12:09:12 +0200 Subject: DBO_rz_clone - reconstruct links from link tuples --- src-py/db_controller.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 40038ed8..273686fd 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -370,7 +370,7 @@ class DBO_rz_clone(DB_op): 'order by n.id', 'skip %d' % (self.skip), 'limit %d' % (self.limit), - 'return n,collect([n.id, m.id, r])'] + 'return n,labels(n),collect([m.id, r, type(r)])'] q = ' '.join(q_arr) self.add_statement(q) @@ -380,17 +380,28 @@ class DBO_rz_clone(DB_op): ret_l_set = [] for _, _, row_set in self: for row in row_set: - itr = iter(row) - n = itr.next() + n, n_lbl_set, l_set = row.items() # see query return statement - l_set = itr.next() - for l in l_set: - assert 3 == len(l) # (n.id, m.id, r) tuples - if None == l[1]: + # reconstruct nodes + assert None != n['id'] + + n['__label_set'] = n_lbl_set + ret_n_set.append(n) + + # reconstruct links from link tuples + for l_tuple in l_set: + assert 3 == len(l_tuple) # see query return statement + + if None == l_tuple[0]: # check if link dst is None # as link matching is optional, collect may yield empty sets continue + + l = l_tuple[1] + l['__src'] = n['id'] + l['__dst'] = l_tuple[0] + l['__label_set'] = [l_tuple[2]] # box single value returned by type() + ret_l_set.append(l) - ret_n_set.append(n) return {'node_set': ret_n_set, 'link_set': ret_l_set } -- cgit v1.3.1 From 48e483a6f7bb11c22b810341088f441706474a06 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 9 Nov 2014 12:12:57 +0200 Subject: misc --- src-py/db_controller.py | 4 ++++ src-py/neo4j_util.py | 19 ++++++++++++++----- src-py/test_db_controller.py | 29 ++++++++++++++++++----------- 3 files changed, 36 insertions(+), 16 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 273686fd..d9d8d737 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -63,6 +63,10 @@ class DB_op(object): def parse_multi_statement_response_data(self, data): pass + @property + def name(self): + return self.__class__.__name__ + def process_result_set(self): """ DB op can issue complex sets of quries all at once - this helper method diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index 642e094a..2b6257b4 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -6,14 +6,17 @@ import json import urllib2 import model import string +import time + +from util import debug_log_duration class Neo4JException(Exception): def __init__(self, error_set): self.error_set = error_set - + def __str__(self): return 'neo4j error set: ' + str(self.error_set) - + class DB_row(object): def __init__(self, data): self.data = data @@ -22,6 +25,9 @@ class DB_row(object): for column_val in self.data: yield column_val + def items(self): + return [x for x in self] + class DB_result_set(object): def __init__(self, data): self.data = data @@ -33,6 +39,9 @@ class DB_result_set(object): yield DB_row(db_row_dict['row']) + def items(self): + return [x for x in self] + class Cypher_String_Formatter(string.Formatter): """ Despite parameter support in Cypher, we sometimes do engage in query string building @@ -206,12 +215,12 @@ def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__type'): """ ret = {} for v in e_set: - assert None != v['__type'] # attert type meta-attr is present + assert None != v['__type'] # assert type meta-attr is present v_type = v['__type'] - if None == ret.get(v_type):# init type list if necessary + if None == ret.get(v_type): # init type list if necessary ret[v_type] = [] - + v_no_meta = v.copy() del v_no_meta['__type'] diff --git a/src-py/test_db_controller.py b/src-py/test_db_controller.py index 4ff0c4de..da8ae364 100644 --- a/src-py/test_db_controller.py +++ b/src-py/test_db_controller.py @@ -5,6 +5,7 @@ import db_controller as dbc from rhizi_server import Config from neo4j_test_util import rand_id from neo4j_test_util import flush_db +from neo4j_test_util import gen_rand_data from neo4j_util import Neo4JException from model.graph import Attr_Diff @@ -35,8 +36,12 @@ class TestDBController(unittest.TestCase): self.log = logging.getLogger('rhizi') self.log.addHandler(logging.StreamHandler()) + # TODO rm when implemented: neo4j_test_util + self.db_ctl.exec_cypher_query('create index on :Person(id)') + self.db_ctl.exec_cypher_query('create index on :Skill(id)') + def setUp(self): - flush_db(self.db_ctl) # remove once embedded DB test mode is supported + flush_db(self.db_ctl) # remove once embedded DB test mode is supported self.db_ctl.exec_op(dbc.DBO_add_node_set(self.n_map)) self.db_ctl.exec_op(dbc.DBO_add_link_set(self.l_map)) @@ -127,6 +132,8 @@ class TestDBController(unittest.TestCase): self.assertEqual(len(id_set), 0) def test_load_link_set(self): + + # load by l_ptr l_ptr = Link.link_ptr(src_id='person_00', dst_id='skill_00') op = dbc.DBO_load_link_set.init_from_link_ptr(l_ptr) l_set = self.db_ctl.exec_op(op) @@ -164,7 +171,7 @@ class TestDBController(unittest.TestCase): op = dbc.DBO_add_node_set({'T_test_load_node_set_by_DB_id': [{'name': 'John Doe'}, {'name': 'John Doe'}]}) id_set = self.db_ctl.exec_op(op) - + # match against DB ids op = dbc.DBO_load_node_set_by_DB_id(id_set) n_set = self.db_ctl.exec_op(op) @@ -219,16 +226,16 @@ class TestDBController(unittest.TestCase): id_set = self.db_ctl.exec_op(dbc.DBO_match_node_set_by_id_attribute([n_0_id, n_1_id])) self.assertEqual(len(id_set), 2) - + l_ptr = Link.link_ptr(src_id=n_0_id, dst_id=n_1_id) id_set = self.db_ctl.exec_op(dbc.DBO_load_link_set.init_from_link_ptr(l_ptr)) self.assertEqual(len(id_set), 1) - + l_ptr = Link.link_ptr(src_id=n_1_id, dst_id=n_0_id) id_set = self.db_ctl.exec_op(dbc.DBO_load_link_set.init_from_link_ptr(l_ptr)) self.assertEqual(len(id_set), 1) - id_set_rm=[n_2_id] + id_set_rm = [n_2_id] topo_diff = Topo_Diff(node_set_rm=id_set_rm) op = dbc.DBO_topo_diff_commit(topo_diff) self.db_ctl.exec_op(op) @@ -278,7 +285,7 @@ class TestDBController(unittest.TestCase): n_2_id = rand_id() n_3_id = rand_id() n_T = 'T_test_rm_node_set' - + n_set = [{'__type': n_T, 'id': n_0_id }, {'__type': n_T, 'id': n_1_id }, {'__type': n_T, 'id': n_2_id }, @@ -291,13 +298,13 @@ class TestDBController(unittest.TestCase): op = dbc.DBO_topo_diff_commit(topo_diff) self.db_ctl.exec_op(op) - + op = dbc.DBO_rm_node_set([n_0_id, n_1_id]) self.db_ctl.exec_op(op) - + op = dbc.DBO_rm_node_set([n_2_id, n_3_id], rm_links=True) self.db_ctl.exec_op(op) - + # assert all deleted op = dbc.DBO_match_node_id_set(filter_type=n_T) id_set = self.db_ctl.exec_op(op) @@ -305,11 +312,11 @@ class TestDBController(unittest.TestCase): def test_rz_clone(self): l_n, l_r = gen_rand_data(self.db_ctl, lim_n=8, lim_r=16, prob_link_create=0.7) - op = dbc.DBO_rz_clone(filter_label = l_n, limit=32) + op = dbc.DBO_rz_clone(filter_label=l_n, limit=32) ret = self.db_ctl.exec_op(op) n_set = ret['node_set'] l_set = ret['link_set'] - + # TODO improve assertions self.assertTrue(0 < len(n_set)) self.assertTrue(0 < len(l_set)) -- cgit v1.3.1 From ed2e0747a050f06e0b392e5aacb21f869eaa544b Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Mon, 24 Nov 2014 12:11:11 +0200 Subject: db_controller: remove unused import --- src-py/db_controller.py | 1 - 1 file changed, 1 deletion(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index d9d8d737..5602efac 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -5,7 +5,6 @@ import logging import os import re import traceback -import urllib2 from db_driver import DB_Driver_REST, DB_Driver_Base from model.graph import Attr_Diff -- cgit v1.3.1 From 1045754c6e1b5587f09fef11be8b62094a29d697 Mon Sep 17 00:00:00 2001 From: Alon Levy Date: Mon, 24 Nov 2014 12:11:36 +0200 Subject: db_controller: white space before EOL removal --- src-py/db_controller.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 5602efac..8800222a 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -43,11 +43,11 @@ class DB_op(object): def __iter__(self): """ - iterate over (statement_index, statement, result, error) + iterate over (statement_index, statement, result, error) where result & error are mutually exclusive - + note: statement_index is zero based - + TODO: handle partial iteration due to error_set being non-empty """ i = 0 @@ -94,7 +94,7 @@ class DB_composed_op(DB_op): def __getattribute__(self, attr): """ - intercept 'statement_set' attr get + intercept 'statement_set' attr get """ if attr == 'statement_set': self.__assert_false_statement_access() @@ -199,7 +199,7 @@ class DBO_add_node_set(DB_op): def __init__(self, node_map): """ DB op: add node set - + @param node_map: node-type to node-set map @return: set of new node DB ids """ @@ -239,7 +239,7 @@ class DBO_load_node_set_by_DB_id(DB_op): def __init__(self, id_set): """ load a set of nodes whose DB id is in id_set - + @param id_set: DB node id set @return: loaded node set or an empty set if no match was found """ @@ -252,11 +252,11 @@ class DBO_match_node_id_set(DB_op): def __init__(self, filter_label=None, filter_attr_map={}): """ match a set of nodes by type / attr_map - + @param filter_label: node type filter @param filter_attr_map: is a filter_key to filter_value_set map of possible attributes to match against, eg.: - { 'id':[0,1], 'color: ['red','blue'] } + { 'id':[0,1], 'color: ['red','blue'] } @return: a set of node DB id's """ super(DBO_match_node_id_set, self).__init__() @@ -283,7 +283,7 @@ class DBO_load_link_set(DB_op): def __init__(self, link_ptr_set): """ match a set of sets of links by source/target node id attributes - + This class should be instantiated through a static factory function @link_ptr_set link pointer set @@ -316,8 +316,8 @@ class DBO_match_link_id_set(DB_op): def __init__(self, filter_label=None, filter_attr_map={}): """ load an id-set of links - - @param filter_label: link type filter + + @param filter_label: link type filter @param filter_attr_map: is a filter_key to filter_value_set map of attributes to match link properties against @return: a set of loaded link ids @@ -357,7 +357,7 @@ class DBO_rz_clone(DB_op): def __init__(self, filter_label=None, limit=128): """ clone rhizi - + @return: a dict: {'node_set': n_set, 'link_set': l_set } where l_set is a list of (src.id, dst.id, link) tuples -- cgit v1.3.1 From 725f5a620ea241f7bca46b5e5f32786634a8e97c Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 8 Dec 2014 22:33:26 +0200 Subject: DBO_rm_node_set: use query param id_set --- src-py/db_controller.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 8800222a..66bc5a47 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -340,18 +340,19 @@ class DBO_rm_node_set(DB_op): if rm_links: q_arr = ['match (n)', - 'where n.id in ' + str(id_set), + 'where n.id in {id_set}', 'optional match (n)-[r]-()', 'delete n,r' ] else: q_arr = ['match (n)', - 'where n.id in ' + str(id_set), + 'where n.id in {id_set}', 'delete n' ] q = ' '.join(q_arr) # TODO: use id param upon neo4j support: q_params = {'id_set': id_set} - self.add_statement(q) + q_params = {'id_set': id_set} + self.add_statement(q, q_params) class DBO_rz_clone(DB_op): def __init__(self, filter_label=None, limit=128): -- cgit v1.3.1 From 1391f30e366df1c14487f5c21f037258a1e219bd Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 10 Dec 2014 15:06:43 +0200 Subject: DBO_rm_link_set --- src-py/db_controller.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 66bc5a47..00043d04 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -130,13 +130,9 @@ class DBO_topo_diff_commit(DB_composed_op): def __init__(self, topo_diff): super(DBO_topo_diff_commit, self).__init__() - # TODO rm link set - # TODO rm node set - assert 0 == len(topo_diff.link_set_rm), 'unsupported' - n_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.node_set_add) l_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.link_set_add) - l_rm_set = [] + l_rm_set = topo_diff.link_set_rm n_rm_set = topo_diff.node_set_rm # @@ -151,7 +147,8 @@ class DBO_topo_diff_commit(DB_composed_op): self.add_sub_op(op) if len(l_rm_set) > 0: - pass + op = DBO_rm_link_set(l_rm_set) + self.add_sub_op(op) if len(n_rm_set) > 0: op = DBO_rm_node_set(n_rm_set) @@ -351,6 +348,25 @@ class DBO_rm_node_set(DB_op): ] q = ' '.join(q_arr) # TODO: use id param upon neo4j support: q_params = {'id_set': id_set} +class DBO_rm_link_set(DB_op): + def __init__(self, id_set): + """ + remove link set + + [!] when removing as a result of node removal, use DBO_rm_node_set + along with rm_links=True + """ + assert len(id_set) > 0, __name__ + ': empty id set' + + super(DBO_rm_link_set, self).__init__() + + q_arr = ['match ()-[r]->()', + 'where r.id in {id_set}', + 'delete r', + 'return {id_set}' + ] + + q = ' '.join(q_arr) q_params = {'id_set': id_set} self.add_statement(q, q_params) -- cgit v1.3.1 From b6cd93638ee183fb120778ec55fca9f15aa9804b Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 10 Dec 2014 15:13:17 +0200 Subject: DBO_rm_node_set: return id_set --- src-py/db_controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 00043d04..c5798aa5 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -344,7 +344,8 @@ class DBO_rm_node_set(DB_op): else: q_arr = ['match (n)', 'where n.id in {id_set}', - 'delete n' + 'delete n', + 'return {id_set}' ] q = ' '.join(q_arr) # TODO: use id param upon neo4j support: q_params = {'id_set': id_set} -- cgit v1.3.1 From 4aa6f7dc89509589fa7641ad95e3abc9ba24d528 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 10 Dec 2014 15:13:51 +0200 Subject: assert len(id_set) > 0 --- src-py/db_controller.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index c5798aa5..1e63b0bc 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -333,6 +333,8 @@ class DBO_rm_node_set(DB_op): """ remove node set """ + assert len(id_set) > 0, __name__ + ': empty id set' + super(DBO_rm_node_set, self).__init__() if rm_links: @@ -348,7 +350,10 @@ class DBO_rm_node_set(DB_op): 'return {id_set}' ] - q = ' '.join(q_arr) # TODO: use id param upon neo4j support: q_params = {'id_set': id_set} + q = ' '.join(q_arr) + q_params = {'id_set': id_set} + self.add_statement(q, q_params) + class DBO_rm_link_set(DB_op): def __init__(self, id_set): """ -- cgit v1.3.1 From 18d96ad7f7b5abd58ccfd485de5d8c20ec3f395b Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 10 Dec 2014 15:15:16 +0200 Subject: switch to q_arr format --- src-py/db_controller.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 1e63b0bc..460686a5 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -321,7 +321,10 @@ class DBO_match_link_id_set(DB_op): """ super(DBO_match_link_id_set, self).__init__() - q = "match ()-[r{filter_label} {filter_attr}]->() return id(r)" + q_arr = ['match ()-[r{filter_label} {filter_attr}]->()', + 'return id(r)' + ] + q = ' '.join(q_arr) q = cfmt(q, filter_label="" if not filter_label else ":" + filter_label) q = cfmt(q, filter_attr=db_util.gen_clause_attr_filter_from_filter_attr_map(filter_attr_map)) q_params = {k: v[0] for (k, v) in filter_attr_map.items()} # pass on only first value from each value set -- cgit v1.3.1 From 0950293605eecff2de9cdadccf1def2d7d9b73bc Mon Sep 17 00:00:00 2001 From: LV-426 Date: Wed, 10 Dec 2014 15:16:06 +0200 Subject: distinguish __src (obj) vs. __src_id (str) --- src-py/db_controller.py | 4 ++-- src-py/model/model.py | 25 ++++++++++++++++--------- src-py/neo4j_util.py | 16 ++++++++-------- src-py/rhizi_api.py | 4 ++-- 4 files changed, 28 insertions(+), 21 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 460686a5..2d4fe8ce 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -426,8 +426,8 @@ class DBO_rz_clone(DB_op): continue l = l_tuple[1] - l['__src'] = n['id'] - l['__dst'] = l_tuple[0] + l['__src_id'] = n['id'] + l['__dst_id'] = l_tuple[0] l['__label_set'] = [l_tuple[2]] # box single value returned by type() ret_l_set.append(l) diff --git a/src-py/model/model.py b/src-py/model/model.py index 701074ab..33cf9e6a 100644 --- a/src-py/model/model.py +++ b/src-py/model/model.py @@ -3,24 +3,31 @@ class Link(): documentation anchor - this class currently carries no implementation and only acts as a documentation anchor - link['__src'] - meta attribute for link source - link['__dst'] - meta attribute for link destination + link['__src'] - meta attribute for link source object + link['__dst'] - meta attribute for link destination object """ - + + def __init__(self, src=None, dst=None): + assert False, 'currently unused' + class Link_Ptr(dict): + """ + link['__src_id'] - meta attribute for link source id + link['__dst_id'] - meta attribute for link destination id + """ def __init__(self, src_id=None, dst_id=None): assert None != src_id or None != dst_id - - self['__src'] = src_id - self['__dst'] = dst_id - + + self['__src_id'] = src_id + self['__dst_id'] = dst_id + @property def src_id(self): - return self['__src'] + return self['__src_id'] @property def dst_id(self): - return self['__dst'] + return self['__dst_id'] @staticmethod def link_ptr(src_id=None, dst_id=None): diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py index cc3d53d8..a203207f 100644 --- a/src-py/neo4j_util.py +++ b/src-py/neo4j_util.py @@ -199,16 +199,16 @@ def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: for link in l_set: __type_check_link(link) - n_src = link['__src'] - n_dst = link['__dst'] + src_id = link['__src_id'] + dst_id = link['__dst_id'] # TODO: use object based link representation prop_dict = link.copy() - del prop_dict['__dst'] - del prop_dict['__src'] + del prop_dict['__dst_id'] + del prop_dict['__src_id'] - q_params = {'src': { 'id': n_src} , - 'dst': { 'id': n_dst} , + q_params = {'src': { 'id': src_id} , + 'dst': { 'id': dst_id} , 'link_attr' : input_to_DB_property_map(prop_dict)} ret.append((q, q_params)) @@ -239,8 +239,8 @@ def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__label_set'): return ret def __type_check_link(link): - assert link.has_key('__src') - assert link.has_key('__dst') + assert link.has_key('__src_id') + assert link.has_key('__dst_id') def __type_check_link_or_node_map(x_map): for k, v in x_map.iteritems(): # do some type sanity checking diff --git a/src-py/rhizi_api.py b/src-py/rhizi_api.py index 8d787a5c..ef904a37 100644 --- a/src-py/rhizi_api.py +++ b/src-py/rhizi_api.py @@ -127,8 +127,8 @@ def load_link_set_by_link_ptr_set(): l_ptr_set = [] for lptr_dict in l_ptr_set_raw: - src_id = lptr_dict.get('__src') - dst_id = lptr_dict.get('__dst') + src_id = lptr_dict.get('__src_id') + dst_id = lptr_dict.get('__dst_id') l_ptr_set += [Link.Link_Ptr(src_id=src_id, dst_id=dst_id) ] return l_ptr_set -- cgit v1.3.1 From a6cd66befa7f9b4041f7b06ecee408dd02617fb2 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 14 Dec 2014 22:41:51 +0200 Subject: complete 'return {id_set}' move --- src-py/db_controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 2d4fe8ce..2ca7b551 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -344,7 +344,8 @@ class DBO_rm_node_set(DB_op): q_arr = ['match (n)', 'where n.id in {id_set}', 'optional match (n)-[r]-()', - 'delete n,r' + 'delete n,r', + 'return {id_set}' ] else: q_arr = ['match (n)', -- cgit v1.3.1 From 43fc57a01028b374234042ab59fdbdd48157a88e Mon Sep 17 00:00:00 2001 From: LV-426 Date: Sun, 14 Dec 2014 22:45:27 +0200 Subject: Attr_Diff: __type_node, __type_link separation --- src-py/db_controller.py | 9 ++++++--- src-py/model/graph.py | 49 ++++++++++++++++++++++++++++++++++++++----------- src-py/rhizi_api.py | 9 +++++++-- 3 files changed, 51 insertions(+), 16 deletions(-) (limited to 'src-py/db_controller.py') diff --git a/src-py/db_controller.py b/src-py/db_controller.py index 2ca7b551..c17fd0cd 100644 --- a/src-py/db_controller.py +++ b/src-py/db_controller.py @@ -161,10 +161,10 @@ class DBO_attr_diff_commit(DB_op): def __init__(self, attr_diff): super(DBO_attr_diff_commit, self).__init__() - for id_attr, n_attr_diff in attr_diff.items(): + for id_attr, n_attr_diff in attr_diff.type__node.items(): # TODO parameterize multiple attr removal - r_attr_set = n_attr_diff['attr_remove'] - w_attr_set = n_attr_diff['attr_write'] + r_attr_set = n_attr_diff['__attr_remove'] + w_attr_set = n_attr_diff['__attr_write'] assert len(r_attr_set) > 0 or len(w_attr_set) > 0 @@ -184,6 +184,9 @@ class DBO_attr_diff_commit(DB_op): q = " ".join(q_arr) self.add_statement(q, q_param_set) + for id_attr, n_attr_diff in attr_diff.type__link.items(): + pass # TODO: handl link attr_diffs + def process_result_set(self): ret = {} for _, _, r_set in self: diff --git a/src-py/model/graph.py b/src-py/model/graph.py index 86a195d4..b40b805a 100644 --- a/src-py/model/graph.py +++ b/src-py/model/graph.py @@ -5,31 +5,58 @@ class Attr_Diff(dict): changed or removed Example: - attr_diff = {n_id: {'attr_write': {'attr_0': 0, - 'attr_1': 'a'}, - 'attr_remove': ['attr_2'] } + attr_diff = {'__type_node' : {n_id: {'__attr_write': {'attr_0': 0, + 'attr_1': 'a'}, + '__attr_remove': ['attr_2'] }} + '__type_link' : {l_id: ... } } """ def __init__(self): - pass + self['__type_node'] = {} + self['__type_link'] = {} def init_node_attr_diff(self, n_id): - ret = {'attr_write': {}, - 'attr_remove': []} - self[n_id] = ret + ret = {'__attr_write': {}, + '__attr_remove': []} + self['__type_node'][n_id] = ret + return ret + + @staticmethod + def from_json_dict(json_dict): + ret = Attr_Diff() + for obj_type in ret.keys(): + obj_ad_set = json_dict.get(obj_type) + if None != obj_ad_set: + for o_id, ad in obj_ad_set.items(): + if None != ad.get('__attr_write'): + for k, v in ad['__attr_write'].items(): + ret.add_node_attr_write(o_id, k, v) + if None != ad.get('__attr_remove'): + for k in ad['__attr_remove']: + ret.add_node_attr_rm(o_id, k) return ret + @property + def type__node(self): + return self['__type_node'] + + @property + def type__link(self): + return self['__type_link'] + def add_node_attr_write(self, n_id, attr_name, attr_val): - n_attr_diff = self.get(n_id) + + + n_attr_diff = self['__type_node'].get(n_id) if None == n_attr_diff: n_attr_diff = self.init_node_attr_diff(n_id) - n_attr_diff['attr_write'][attr_name] = attr_val + n_attr_diff['__attr_write'][attr_name] = attr_val def add_node_attr_rm(self, n_id, attr_name): - n_attr_diff = self.get(n_id) + n_attr_diff = self['__type_node'].get(n_id) if None == n_attr_diff: n_attr_diff = self.init_node_attr_diff(n_id) - n_attr_diff['attr_remove'].append(attr_name) + n_attr_diff['__attr_remove'].append(attr_name) class Topo_Diff(object): """ diff --git a/src-py/rhizi_api.py b/src-py/rhizi_api.py index 0c1b68b5..b89e8c18 100644 --- a/src-py/rhizi_api.py +++ b/src-py/rhizi_api.py @@ -179,9 +179,14 @@ def diff_commit__attr(): """ commit a graph attribute diff """ - attr_diff = request.get_json()['attr_diff'] - __sanitize_input(attr_diff) + def sanitize_input(req): + attr_diff_dict = request.get_json()['attr_diff'] + attr_diff = Attr_Diff.from_json_dict(attr_diff_dict) + + sanitize_input__attr_diff(attr_diff) + return attr_diff; + attr_diff = sanitize_input(request) op = dbc.DBO_attr_diff_commit(attr_diff) return __common_exec(op) -- cgit v1.3.1